2024-11-19 12:13:16,608 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-19 12:13:16,624 main DEBUG Took 0.013904 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-19 12:13:16,625 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-19 12:13:16,625 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-19 12:13:16,627 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-19 12:13:16,628 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:13:16,638 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-19 12:13:16,654 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:13:16,656 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:13:16,657 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:13:16,658 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:13:16,658 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:13:16,659 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:13:16,660 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:13:16,661 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:13:16,661 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:13:16,662 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:13:16,663 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:13:16,664 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:13:16,665 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:13:16,665 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:13:16,666 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:13:16,666 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:13:16,667 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:13:16,667 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:13:16,668 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:13:16,669 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:13:16,669 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:13:16,670 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:13:16,671 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:13:16,671 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:13:16,672 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:13:16,672 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-19 12:13:16,674 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:13:16,676 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-19 12:13:16,679 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-19 12:13:16,680 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-19 12:13:16,682 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-19 12:13:16,682 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-19 12:13:16,694 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-19 12:13:16,697 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-19 12:13:16,698 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-19 12:13:16,699 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-19 12:13:16,699 main DEBUG createAppenders(={Console}) 2024-11-19 12:13:16,700 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-19 12:13:16,700 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-19 12:13:16,700 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-19 12:13:16,701 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-19 12:13:16,701 main DEBUG OutputStream closed 2024-11-19 12:13:16,701 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-19 12:13:16,702 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-19 12:13:16,703 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-19 12:13:16,840 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-19 12:13:16,844 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-19 12:13:16,847 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-19 12:13:16,848 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-19 12:13:16,849 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-19 12:13:16,850 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-19 12:13:16,851 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-19 12:13:16,856 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-19 12:13:16,857 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-19 12:13:16,858 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-19 12:13:16,858 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-19 12:13:16,859 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-19 12:13:16,860 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-19 12:13:16,860 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-19 12:13:16,861 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-19 12:13:16,861 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-19 12:13:16,862 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-19 12:13:16,864 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-19 12:13:16,868 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19 12:13:16,868 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-19 12:13:16,869 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-19 12:13:16,870 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-19T12:13:16,887 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-19 12:13:16,891 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-19 12:13:16,891 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19T12:13:17,291 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138 2024-11-19T12:13:17,320 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-19T12:13:17,340 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/cluster_b1509a57-692b-6599-da25-cf13fde708a7, deleteOnExit=true 2024-11-19T12:13:17,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-19T12:13:17,345 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/test.cache.data in system properties and HBase conf 2024-11-19T12:13:17,346 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/hadoop.tmp.dir in system properties and HBase conf 2024-11-19T12:13:17,347 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/hadoop.log.dir in system properties and HBase conf 2024-11-19T12:13:17,349 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-19T12:13:17,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-19T12:13:17,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-19T12:13:17,462 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-19T12:13:17,553 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:13:17,557 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-19T12:13:17,557 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-19T12:13:17,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-19T12:13:17,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T12:13:17,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-19T12:13:17,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-19T12:13:17,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T12:13:17,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T12:13:17,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-19T12:13:17,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/nfs.dump.dir in system properties and HBase conf 2024-11-19T12:13:17,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/java.io.tmpdir in system properties and HBase conf 2024-11-19T12:13:17,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T12:13:17,562 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-19T12:13:17,562 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-19T12:13:19,000 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-19T12:13:19,068 INFO [Time-limited test {}] log.Log(170): Logging initialized @3365ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-19T12:13:19,133 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T12:13:19,189 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T12:13:19,211 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T12:13:19,212 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T12:13:19,213 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-19T12:13:19,226 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T12:13:19,229 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4d9452bf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/hadoop.log.dir/,AVAILABLE} 2024-11-19T12:13:19,230 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5d5b118c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T12:13:19,410 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3260f08c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/java.io.tmpdir/jetty-localhost-39841-hadoop-hdfs-3_4_1-tests_jar-_-any-15185476289601292016/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T12:13:19,421 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@369f03e8{HTTP/1.1, (http/1.1)}{localhost:39841} 2024-11-19T12:13:19,422 INFO [Time-limited test {}] server.Server(415): Started @3719ms 2024-11-19T12:13:20,346 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T12:13:20,354 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T12:13:20,355 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T12:13:20,355 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T12:13:20,356 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T12:13:20,356 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20ec9bac{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/hadoop.log.dir/,AVAILABLE} 2024-11-19T12:13:20,357 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5026a795{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T12:13:20,456 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4f33137f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/java.io.tmpdir/jetty-localhost-36655-hadoop-hdfs-3_4_1-tests_jar-_-any-18241740158946281446/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T12:13:20,457 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6de9cba2{HTTP/1.1, (http/1.1)}{localhost:36655} 2024-11-19T12:13:20,457 INFO [Time-limited test {}] server.Server(415): Started @4755ms 2024-11-19T12:13:20,503 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-19T12:13:20,607 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T12:13:20,615 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T12:13:20,619 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T12:13:20,619 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T12:13:20,620 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T12:13:20,622 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5b3caf0c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/hadoop.log.dir/,AVAILABLE} 2024-11-19T12:13:20,623 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@540e5e4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T12:13:20,746 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7549e2f1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/java.io.tmpdir/jetty-localhost-43777-hadoop-hdfs-3_4_1-tests_jar-_-any-7013331831837532758/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T12:13:20,746 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@53e0679{HTTP/1.1, (http/1.1)}{localhost:43777} 2024-11-19T12:13:20,747 INFO [Time-limited test {}] server.Server(415): Started @5044ms 2024-11-19T12:13:20,749 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-19T12:13:20,787 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T12:13:20,792 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T12:13:20,796 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T12:13:20,796 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T12:13:20,796 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-19T12:13:20,797 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@469f5a42{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/hadoop.log.dir/,AVAILABLE} 2024-11-19T12:13:20,798 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75445ba5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T12:13:20,895 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6eecab6b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/java.io.tmpdir/jetty-localhost-33183-hadoop-hdfs-3_4_1-tests_jar-_-any-17869280591437888196/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T12:13:20,896 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3fb3fd4b{HTTP/1.1, (http/1.1)}{localhost:33183} 2024-11-19T12:13:20,896 INFO [Time-limited test {}] server.Server(415): Started @5194ms 2024-11-19T12:13:20,898 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-19T12:13:22,583 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/cluster_b1509a57-692b-6599-da25-cf13fde708a7/data/data1/current/BP-993660459-172.17.0.2-1732018398433/current, will proceed with Du for space computation calculation, 2024-11-19T12:13:22,583 WARN [Thread-126 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/cluster_b1509a57-692b-6599-da25-cf13fde708a7/data/data3/current/BP-993660459-172.17.0.2-1732018398433/current, will proceed with Du for space computation calculation, 2024-11-19T12:13:22,583 WARN [Thread-128 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/cluster_b1509a57-692b-6599-da25-cf13fde708a7/data/data5/current/BP-993660459-172.17.0.2-1732018398433/current, will proceed with Du for space computation calculation, 2024-11-19T12:13:22,583 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/cluster_b1509a57-692b-6599-da25-cf13fde708a7/data/data2/current/BP-993660459-172.17.0.2-1732018398433/current, will proceed with Du for space computation calculation, 2024-11-19T12:13:22,583 WARN [Thread-129 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/cluster_b1509a57-692b-6599-da25-cf13fde708a7/data/data6/current/BP-993660459-172.17.0.2-1732018398433/current, will proceed with Du for space computation calculation, 2024-11-19T12:13:22,583 WARN [Thread-127 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/cluster_b1509a57-692b-6599-da25-cf13fde708a7/data/data4/current/BP-993660459-172.17.0.2-1732018398433/current, will proceed with Du for space computation calculation, 2024-11-19T12:13:22,626 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-19T12:13:22,626 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-19T12:13:22,629 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-19T12:13:22,673 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb9d0cd6623674913 with lease ID 0xbb5cad71d0ccfb10: Processing first storage report for DS-598e6c09-e04f-4361-a363-9b507e9888f8 from datanode DatanodeRegistration(127.0.0.1:46231, datanodeUuid=57507d16-d066-42fa-89a4-91bf3e194ef8, infoPort=40699, infoSecurePort=0, ipcPort=35993, storageInfo=lv=-57;cid=testClusterID;nsid=141479024;c=1732018398433) 2024-11-19T12:13:22,674 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb9d0cd6623674913 with lease ID 0xbb5cad71d0ccfb10: from storage DS-598e6c09-e04f-4361-a363-9b507e9888f8 node DatanodeRegistration(127.0.0.1:46231, datanodeUuid=57507d16-d066-42fa-89a4-91bf3e194ef8, infoPort=40699, infoSecurePort=0, ipcPort=35993, storageInfo=lv=-57;cid=testClusterID;nsid=141479024;c=1732018398433), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-19T12:13:22,675 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x10b78ec14c157de1 with lease ID 0xbb5cad71d0ccfb11: Processing first storage report for DS-fc24d53f-eb0d-4d9b-b2d4-feb49c03511f from datanode DatanodeRegistration(127.0.0.1:40579, datanodeUuid=9007d91f-70e9-4451-a95d-05fbe4a7ea02, infoPort=39493, infoSecurePort=0, ipcPort=40625, storageInfo=lv=-57;cid=testClusterID;nsid=141479024;c=1732018398433) 2024-11-19T12:13:22,675 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x10b78ec14c157de1 with lease ID 0xbb5cad71d0ccfb11: from storage DS-fc24d53f-eb0d-4d9b-b2d4-feb49c03511f node DatanodeRegistration(127.0.0.1:40579, datanodeUuid=9007d91f-70e9-4451-a95d-05fbe4a7ea02, infoPort=39493, infoSecurePort=0, ipcPort=40625, storageInfo=lv=-57;cid=testClusterID;nsid=141479024;c=1732018398433), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-19T12:13:22,675 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x877d31aa807e063c with lease ID 0xbb5cad71d0ccfb0f: Processing first storage report for DS-73e25d91-6f88-4582-919b-1060a325c9ba from datanode DatanodeRegistration(127.0.0.1:46857, datanodeUuid=6304d379-6db4-4905-89b2-49fd804d188c, infoPort=45779, infoSecurePort=0, ipcPort=41039, storageInfo=lv=-57;cid=testClusterID;nsid=141479024;c=1732018398433) 2024-11-19T12:13:22,675 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x877d31aa807e063c with lease ID 0xbb5cad71d0ccfb0f: from storage DS-73e25d91-6f88-4582-919b-1060a325c9ba node DatanodeRegistration(127.0.0.1:46857, datanodeUuid=6304d379-6db4-4905-89b2-49fd804d188c, infoPort=45779, infoSecurePort=0, ipcPort=41039, storageInfo=lv=-57;cid=testClusterID;nsid=141479024;c=1732018398433), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-19T12:13:22,675 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb9d0cd6623674913 with lease ID 0xbb5cad71d0ccfb10: Processing first storage report for DS-a53359da-3fd3-4b96-b579-b59e1d8a9301 from datanode DatanodeRegistration(127.0.0.1:46231, datanodeUuid=57507d16-d066-42fa-89a4-91bf3e194ef8, infoPort=40699, infoSecurePort=0, ipcPort=35993, storageInfo=lv=-57;cid=testClusterID;nsid=141479024;c=1732018398433) 2024-11-19T12:13:22,676 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb9d0cd6623674913 with lease ID 0xbb5cad71d0ccfb10: from storage DS-a53359da-3fd3-4b96-b579-b59e1d8a9301 node DatanodeRegistration(127.0.0.1:46231, datanodeUuid=57507d16-d066-42fa-89a4-91bf3e194ef8, infoPort=40699, infoSecurePort=0, ipcPort=35993, storageInfo=lv=-57;cid=testClusterID;nsid=141479024;c=1732018398433), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-19T12:13:22,676 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x10b78ec14c157de1 with lease ID 0xbb5cad71d0ccfb11: Processing first storage report for DS-9c5f4f64-2ae2-455b-9dd9-ffe4b89fcd78 from datanode DatanodeRegistration(127.0.0.1:40579, datanodeUuid=9007d91f-70e9-4451-a95d-05fbe4a7ea02, infoPort=39493, infoSecurePort=0, ipcPort=40625, storageInfo=lv=-57;cid=testClusterID;nsid=141479024;c=1732018398433) 2024-11-19T12:13:22,676 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x10b78ec14c157de1 with lease ID 0xbb5cad71d0ccfb11: from storage DS-9c5f4f64-2ae2-455b-9dd9-ffe4b89fcd78 node DatanodeRegistration(127.0.0.1:40579, datanodeUuid=9007d91f-70e9-4451-a95d-05fbe4a7ea02, infoPort=39493, infoSecurePort=0, ipcPort=40625, storageInfo=lv=-57;cid=testClusterID;nsid=141479024;c=1732018398433), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-19T12:13:22,676 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x877d31aa807e063c with lease ID 0xbb5cad71d0ccfb0f: Processing first storage report for DS-982314ab-3f9f-4377-ae6a-b8d9e26c26da from datanode DatanodeRegistration(127.0.0.1:46857, datanodeUuid=6304d379-6db4-4905-89b2-49fd804d188c, infoPort=45779, infoSecurePort=0, ipcPort=41039, storageInfo=lv=-57;cid=testClusterID;nsid=141479024;c=1732018398433) 2024-11-19T12:13:22,676 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x877d31aa807e063c with lease ID 0xbb5cad71d0ccfb0f: from storage DS-982314ab-3f9f-4377-ae6a-b8d9e26c26da node DatanodeRegistration(127.0.0.1:46857, datanodeUuid=6304d379-6db4-4905-89b2-49fd804d188c, infoPort=45779, infoSecurePort=0, ipcPort=41039, storageInfo=lv=-57;cid=testClusterID;nsid=141479024;c=1732018398433), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-19T12:13:22,730 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138 2024-11-19T12:13:22,801 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/cluster_b1509a57-692b-6599-da25-cf13fde708a7/zookeeper_0, clientPort=57377, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/cluster_b1509a57-692b-6599-da25-cf13fde708a7/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/bb8a437b-4ae9-3f27-a67e-99a90b451138/cluster_b1509a57-692b-6599-da25-cf13fde708a7/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T12:13:22,813 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57377 2024-11-19T12:13:22,829 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T12:13:22,833 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T12:13:23,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741825_1001 (size=7) 2024-11-19T12:13:23,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741825_1001 (size=7) 2024-11-19T12:13:23,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741825_1001 (size=7) 2024-11-19T12:13:23,464 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 with version=8 2024-11-19T12:13:23,464 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/hbase-staging 2024-11-19T12:13:23,538 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-19T12:13:23,768 INFO [Time-limited test {}] client.ConnectionUtils(128): master/351a98347401:0 server-side Connection retries=45 2024-11-19T12:13:23,777 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:13:23,778 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T12:13:23,782 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T12:13:23,783 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:13:23,783 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T12:13:23,908 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T12:13:23,963 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-19T12:13:23,972 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-19T12:13:23,975 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T12:13:24,001 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2332 (auto-detected) 2024-11-19T12:13:24,002 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-19T12:13:24,021 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38839 2024-11-19T12:13:24,044 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38839 connecting to ZooKeeper ensemble=127.0.0.1:57377 2024-11-19T12:13:24,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:388390x0, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T12:13:24,200 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38839-0x1015303718b0000 connected 2024-11-19T12:13:24,292 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T12:13:24,299 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T12:13:24,313 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:13:24,318 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8, hbase.cluster.distributed=false 2024-11-19T12:13:24,344 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T12:13:24,349 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38839 2024-11-19T12:13:24,350 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38839 2024-11-19T12:13:24,350 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38839 2024-11-19T12:13:24,351 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38839 2024-11-19T12:13:24,351 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38839 2024-11-19T12:13:24,475 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/351a98347401:0 server-side Connection retries=45 2024-11-19T12:13:24,477 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:13:24,478 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T12:13:24,478 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T12:13:24,478 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:13:24,478 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T12:13:24,481 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T12:13:24,484 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T12:13:24,485 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36275 2024-11-19T12:13:24,487 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36275 connecting to ZooKeeper ensemble=127.0.0.1:57377 2024-11-19T12:13:24,488 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T12:13:24,490 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T12:13:24,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:362750x0, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T12:13:24,502 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:362750x0, quorum=127.0.0.1:57377, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:13:24,502 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36275-0x1015303718b0001 connected 2024-11-19T12:13:24,508 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T12:13:24,517 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T12:13:24,520 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T12:13:24,526 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T12:13:24,527 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36275 2024-11-19T12:13:24,527 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36275 2024-11-19T12:13:24,528 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36275 2024-11-19T12:13:24,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36275 2024-11-19T12:13:24,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36275 2024-11-19T12:13:24,556 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/351a98347401:0 server-side Connection retries=45 2024-11-19T12:13:24,556 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:13:24,557 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T12:13:24,557 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T12:13:24,557 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:13:24,558 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T12:13:24,558 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T12:13:24,558 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T12:13:24,559 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45367 2024-11-19T12:13:24,562 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45367 connecting to ZooKeeper ensemble=127.0.0.1:57377 2024-11-19T12:13:24,563 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T12:13:24,566 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T12:13:24,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:453670x0, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T12:13:24,583 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:453670x0, quorum=127.0.0.1:57377, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:13:24,584 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45367-0x1015303718b0002 connected 2024-11-19T12:13:24,584 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T12:13:24,585 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T12:13:24,586 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T12:13:24,588 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T12:13:24,591 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45367 2024-11-19T12:13:24,591 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45367 2024-11-19T12:13:24,592 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45367 2024-11-19T12:13:24,594 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45367 2024-11-19T12:13:24,595 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45367 2024-11-19T12:13:24,610 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/351a98347401:0 server-side Connection retries=45 2024-11-19T12:13:24,610 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:13:24,610 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T12:13:24,610 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T12:13:24,610 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:13:24,611 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T12:13:24,611 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T12:13:24,611 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T12:13:24,612 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35543 2024-11-19T12:13:24,614 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35543 connecting to ZooKeeper ensemble=127.0.0.1:57377 2024-11-19T12:13:24,615 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T12:13:24,617 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T12:13:24,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:355430x0, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T12:13:24,636 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35543-0x1015303718b0003, quorum=127.0.0.1:57377, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:13:24,636 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35543-0x1015303718b0003 connected 2024-11-19T12:13:24,637 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T12:13:24,638 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T12:13:24,640 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35543-0x1015303718b0003, quorum=127.0.0.1:57377, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T12:13:24,641 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35543-0x1015303718b0003, quorum=127.0.0.1:57377, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T12:13:24,644 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35543 2024-11-19T12:13:24,648 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35543 2024-11-19T12:13:24,649 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35543 2024-11-19T12:13:24,650 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35543 2024-11-19T12:13:24,650 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35543 2024-11-19T12:13:24,665 DEBUG [M:0;351a98347401:38839 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;351a98347401:38839 2024-11-19T12:13:24,666 INFO [master/351a98347401:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/351a98347401,38839,1732018403620 2024-11-19T12:13:24,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:13:24,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:13:24,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:13:24,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35543-0x1015303718b0003, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:13:24,682 DEBUG [master/351a98347401:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/351a98347401,38839,1732018403620 2024-11-19T12:13:24,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35543-0x1015303718b0003, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T12:13:24,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T12:13:24,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T12:13:24,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35543-0x1015303718b0003, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:24,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:24,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:24,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:24,713 DEBUG [master/351a98347401:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T12:13:24,714 INFO [master/351a98347401:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/351a98347401,38839,1732018403620 from backup master directory 2024-11-19T12:13:24,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35543-0x1015303718b0003, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:13:24,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/351a98347401,38839,1732018403620 2024-11-19T12:13:24,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:13:24,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:13:24,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:13:24,722 WARN [master/351a98347401:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T12:13:24,723 INFO [master/351a98347401:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=351a98347401,38839,1732018403620 2024-11-19T12:13:24,725 INFO [master/351a98347401:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-19T12:13:24,727 INFO [master/351a98347401:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-19T12:13:24,780 DEBUG [master/351a98347401:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/hbase.id] with ID: 065e031d-65df-4d23-b424-83fdc49a195f 2024-11-19T12:13:24,780 DEBUG [master/351a98347401:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/.tmp/hbase.id 2024-11-19T12:13:24,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741826_1002 (size=42) 2024-11-19T12:13:24,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741826_1002 (size=42) 2024-11-19T12:13:24,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741826_1002 (size=42) 2024-11-19T12:13:24,797 DEBUG [master/351a98347401:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/.tmp/hbase.id]:[hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/hbase.id] 2024-11-19T12:13:24,841 INFO [master/351a98347401:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T12:13:24,847 INFO [master/351a98347401:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T12:13:24,865 INFO [master/351a98347401:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-19T12:13:24,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:24,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:24,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:24,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35543-0x1015303718b0003, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:24,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741827_1003 (size=196) 2024-11-19T12:13:24,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741827_1003 (size=196) 2024-11-19T12:13:24,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741827_1003 (size=196) 2024-11-19T12:13:24,917 INFO [master/351a98347401:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T12:13:24,919 INFO [master/351a98347401:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T12:13:24,931 DEBUG [master/351a98347401:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:13:24,935 INFO [master/351a98347401:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T12:13:24,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741828_1004 (size=1189) 2024-11-19T12:13:24,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741828_1004 (size=1189) 2024-11-19T12:13:24,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741828_1004 (size=1189) 2024-11-19T12:13:24,981 INFO [master/351a98347401:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/MasterData/data/master/store 2024-11-19T12:13:25,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741829_1005 (size=34) 2024-11-19T12:13:25,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741829_1005 (size=34) 2024-11-19T12:13:25,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741829_1005 (size=34) 2024-11-19T12:13:25,008 INFO [master/351a98347401:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-19T12:13:25,011 DEBUG [master/351a98347401:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:25,012 DEBUG [master/351a98347401:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T12:13:25,012 INFO [master/351a98347401:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:13:25,013 DEBUG [master/351a98347401:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:13:25,014 DEBUG [master/351a98347401:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T12:13:25,014 DEBUG [master/351a98347401:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:13:25,014 INFO [master/351a98347401:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:13:25,015 DEBUG [master/351a98347401:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732018405012Disabling compacts and flushes for region at 1732018405012Disabling writes for close at 1732018405014 (+2 ms)Writing region close event to WAL at 1732018405014Closed at 1732018405014 2024-11-19T12:13:25,017 WARN [master/351a98347401:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/MasterData/data/master/store/.initializing 2024-11-19T12:13:25,017 DEBUG [master/351a98347401:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/MasterData/WALs/351a98347401,38839,1732018403620 2024-11-19T12:13:25,024 INFO [master/351a98347401:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T12:13:25,038 INFO [master/351a98347401:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=351a98347401%2C38839%2C1732018403620, suffix=, logDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/MasterData/WALs/351a98347401,38839,1732018403620, archiveDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/MasterData/oldWALs, maxLogs=10 2024-11-19T12:13:25,058 DEBUG [master/351a98347401:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/MasterData/WALs/351a98347401,38839,1732018403620/351a98347401%2C38839%2C1732018403620.1732018405042, exclude list is [], retry=0 2024-11-19T12:13:25,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40579,DS-fc24d53f-eb0d-4d9b-b2d4-feb49c03511f,DISK] 2024-11-19T12:13:25,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46857,DS-73e25d91-6f88-4582-919b-1060a325c9ba,DISK] 2024-11-19T12:13:25,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46231,DS-598e6c09-e04f-4361-a363-9b507e9888f8,DISK] 2024-11-19T12:13:25,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-19T12:13:25,113 INFO [master/351a98347401:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/MasterData/WALs/351a98347401,38839,1732018403620/351a98347401%2C38839%2C1732018403620.1732018405042 2024-11-19T12:13:25,114 DEBUG [master/351a98347401:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45779:45779),(127.0.0.1/127.0.0.1:40699:40699),(127.0.0.1/127.0.0.1:39493:39493)] 2024-11-19T12:13:25,114 DEBUG [master/351a98347401:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:13:25,115 DEBUG [master/351a98347401:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:25,117 DEBUG [master/351a98347401:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:13:25,118 DEBUG [master/351a98347401:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:13:25,153 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:13:25,177 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T12:13:25,180 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:25,183 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:13:25,183 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:13:25,188 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T12:13:25,188 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:25,189 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:25,189 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:13:25,192 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T12:13:25,192 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:25,193 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:25,194 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:13:25,196 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T12:13:25,196 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:25,197 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:25,198 DEBUG [master/351a98347401:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:13:25,201 DEBUG [master/351a98347401:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:13:25,203 DEBUG [master/351a98347401:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:13:25,208 DEBUG [master/351a98347401:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:13:25,208 DEBUG [master/351a98347401:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:13:25,211 DEBUG [master/351a98347401:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T12:13:25,215 DEBUG [master/351a98347401:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:13:25,220 DEBUG [master/351a98347401:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:25,222 INFO [master/351a98347401:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73837232, jitterRate=0.10026049613952637}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T12:13:25,230 DEBUG [master/351a98347401:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732018405130Initializing all the Stores at 1732018405132 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732018405132Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018405133 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018405133Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018405133Cleaning up temporary data from old regions at 1732018405208 (+75 ms)Region opened successfully at 1732018405230 (+22 ms) 2024-11-19T12:13:25,231 INFO [master/351a98347401:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T12:13:25,263 DEBUG [master/351a98347401:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c8077b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=351a98347401/172.17.0.2:0 2024-11-19T12:13:25,290 INFO [master/351a98347401:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T12:13:25,300 INFO [master/351a98347401:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:25,300 INFO [master/351a98347401:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:25,302 INFO [master/351a98347401:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:25,303 INFO [master/351a98347401:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-19T12:13:25,308 INFO [master/351a98347401:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-19T12:13:25,308 INFO [master/351a98347401:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T12:13:25,331 INFO [master/351a98347401:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T12:13:25,338 DEBUG [master/351a98347401:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-19T12:13:25,385 DEBUG [master/351a98347401:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-19T12:13:25,389 INFO [master/351a98347401:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T12:13:25,391 DEBUG [master/351a98347401:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-19T12:13:25,403 DEBUG [master/351a98347401:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-19T12:13:25,405 INFO [master/351a98347401:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T12:13:25,409 DEBUG [master/351a98347401:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-19T12:13:25,416 DEBUG [master/351a98347401:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-19T12:13:25,418 DEBUG [master/351a98347401:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-19T12:13:25,427 DEBUG [master/351a98347401:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-19T12:13:25,450 DEBUG [master/351a98347401:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T12:13:25,458 DEBUG [master/351a98347401:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-19T12:13:25,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T12:13:25,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T12:13:25,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35543-0x1015303718b0003, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T12:13:25,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T12:13:25,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:25,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:25,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35543-0x1015303718b0003, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:25,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:25,474 INFO [master/351a98347401:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=351a98347401,38839,1732018403620, sessionid=0x1015303718b0000, setting cluster-up flag (Was=false) 2024-11-19T12:13:25,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:25,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:25,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35543-0x1015303718b0003, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:25,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:25,532 DEBUG [master/351a98347401:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-19T12:13:25,534 DEBUG [master/351a98347401:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=351a98347401,38839,1732018403620 2024-11-19T12:13:25,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:25,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35543-0x1015303718b0003, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:25,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:25,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:25,585 DEBUG [master/351a98347401:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-19T12:13:25,588 DEBUG [master/351a98347401:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=351a98347401,38839,1732018403620 2024-11-19T12:13:25,596 INFO [master/351a98347401:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T12:13:25,654 INFO [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(746): ClusterId : 065e031d-65df-4d23-b424-83fdc49a195f 2024-11-19T12:13:25,654 INFO [RS:2;351a98347401:35543 {}] regionserver.HRegionServer(746): ClusterId : 065e031d-65df-4d23-b424-83fdc49a195f 2024-11-19T12:13:25,654 INFO [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(746): ClusterId : 065e031d-65df-4d23-b424-83fdc49a195f 2024-11-19T12:13:25,656 DEBUG [RS:0;351a98347401:36275 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T12:13:25,656 DEBUG [RS:1;351a98347401:45367 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T12:13:25,656 DEBUG [RS:2;351a98347401:35543 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T12:13:25,663 DEBUG [master/351a98347401:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:25,671 DEBUG [RS:0;351a98347401:36275 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T12:13:25,671 DEBUG [RS:2;351a98347401:35543 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T12:13:25,671 DEBUG [RS:1;351a98347401:45367 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T12:13:25,671 DEBUG [RS:2;351a98347401:35543 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T12:13:25,671 DEBUG [RS:0;351a98347401:36275 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T12:13:25,671 DEBUG [RS:1;351a98347401:45367 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T12:13:25,671 INFO [master/351a98347401:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T12:13:25,677 INFO [master/351a98347401:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T12:13:25,682 DEBUG [master/351a98347401:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 351a98347401,38839,1732018403620 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T12:13:25,691 DEBUG [RS:2;351a98347401:35543 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T12:13:25,691 DEBUG [RS:1;351a98347401:45367 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T12:13:25,691 DEBUG [RS:0;351a98347401:36275 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T12:13:25,691 DEBUG [RS:1;351a98347401:45367 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3867815f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=351a98347401/172.17.0.2:0 2024-11-19T12:13:25,691 DEBUG [RS:0;351a98347401:36275 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6757dcf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=351a98347401/172.17.0.2:0 2024-11-19T12:13:25,691 DEBUG [RS:2;351a98347401:35543 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@670aff8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=351a98347401/172.17.0.2:0 2024-11-19T12:13:25,693 DEBUG [master/351a98347401:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/351a98347401:0, corePoolSize=5, maxPoolSize=5 2024-11-19T12:13:25,693 DEBUG [master/351a98347401:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/351a98347401:0, corePoolSize=5, maxPoolSize=5 2024-11-19T12:13:25,693 DEBUG [master/351a98347401:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/351a98347401:0, corePoolSize=5, maxPoolSize=5 2024-11-19T12:13:25,693 DEBUG [master/351a98347401:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/351a98347401:0, corePoolSize=5, maxPoolSize=5 2024-11-19T12:13:25,693 DEBUG [master/351a98347401:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/351a98347401:0, corePoolSize=10, maxPoolSize=10 2024-11-19T12:13:25,694 DEBUG [master/351a98347401:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,694 DEBUG [master/351a98347401:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/351a98347401:0, corePoolSize=2, maxPoolSize=2 2024-11-19T12:13:25,694 DEBUG [master/351a98347401:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,700 INFO [master/351a98347401:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018435700 2024-11-19T12:13:25,702 INFO [master/351a98347401:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T12:13:25,703 INFO [master/351a98347401:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T12:13:25,707 DEBUG [RS:1;351a98347401:45367 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;351a98347401:45367 2024-11-19T12:13:25,707 INFO [master/351a98347401:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T12:13:25,707 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:25,708 INFO [master/351a98347401:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T12:13:25,708 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T12:13:25,708 INFO [master/351a98347401:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T12:13:25,708 INFO [master/351a98347401:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T12:13:25,709 DEBUG [RS:2;351a98347401:35543 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;351a98347401:35543 2024-11-19T12:13:25,711 INFO [RS:1;351a98347401:45367 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T12:13:25,711 INFO [RS:2;351a98347401:35543 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T12:13:25,711 INFO [RS:1;351a98347401:45367 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T12:13:25,711 INFO [RS:2;351a98347401:35543 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T12:13:25,711 DEBUG [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T12:13:25,711 DEBUG [RS:2;351a98347401:35543 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T12:13:25,712 INFO [master/351a98347401:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,714 INFO [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(2659): reportForDuty to master=351a98347401,38839,1732018403620 with port=45367, startcode=1732018404555 2024-11-19T12:13:25,714 INFO [RS:2;351a98347401:35543 {}] regionserver.HRegionServer(2659): reportForDuty to master=351a98347401,38839,1732018403620 with port=35543, startcode=1732018404609 2024-11-19T12:13:25,714 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:25,715 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T12:13:25,716 INFO [master/351a98347401:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T12:13:25,718 DEBUG [RS:0;351a98347401:36275 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;351a98347401:36275 2024-11-19T12:13:25,718 INFO [master/351a98347401:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T12:13:25,718 INFO [RS:0;351a98347401:36275 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T12:13:25,718 INFO [RS:0;351a98347401:36275 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T12:13:25,718 DEBUG [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T12:13:25,718 INFO [master/351a98347401:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T12:13:25,719 INFO [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(2659): reportForDuty to master=351a98347401,38839,1732018403620 with port=36275, startcode=1732018404438 2024-11-19T12:13:25,721 INFO [master/351a98347401:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T12:13:25,721 INFO [master/351a98347401:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T12:13:25,723 DEBUG [master/351a98347401:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/351a98347401:0:becomeActiveMaster-HFileCleaner.large.0-1732018405722,5,FailOnTimeoutGroup] 2024-11-19T12:13:25,727 DEBUG [RS:0;351a98347401:36275 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T12:13:25,727 DEBUG [RS:2;351a98347401:35543 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T12:13:25,727 DEBUG [RS:1;351a98347401:45367 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T12:13:25,727 DEBUG [master/351a98347401:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/351a98347401:0:becomeActiveMaster-HFileCleaner.small.0-1732018405723,5,FailOnTimeoutGroup] 2024-11-19T12:13:25,728 INFO [master/351a98347401:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,728 INFO [master/351a98347401:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T12:13:25,729 INFO [master/351a98347401:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,729 INFO [master/351a98347401:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741831_1007 (size=1321) 2024-11-19T12:13:25,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741831_1007 (size=1321) 2024-11-19T12:13:25,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741831_1007 (size=1321) 2024-11-19T12:13:25,739 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T12:13:25,739 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:25,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741832_1008 (size=32) 2024-11-19T12:13:25,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741832_1008 (size=32) 2024-11-19T12:13:25,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741832_1008 (size=32) 2024-11-19T12:13:25,789 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:25,789 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50179, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T12:13:25,789 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37367, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T12:13:25,790 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53003, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T12:13:25,793 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T12:13:25,797 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T12:13:25,798 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:25,798 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 351a98347401,36275,1732018404438 2024-11-19T12:13:25,799 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:13:25,799 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T12:13:25,801 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] master.ServerManager(517): Registering regionserver=351a98347401,36275,1732018404438 2024-11-19T12:13:25,802 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T12:13:25,803 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:25,804 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:13:25,804 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T12:13:25,807 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T12:13:25,808 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:25,809 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:13:25,809 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T12:13:25,813 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T12:13:25,813 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:25,814 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:13:25,815 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T12:13:25,815 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 351a98347401,45367,1732018404555 2024-11-19T12:13:25,815 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] master.ServerManager(517): Registering regionserver=351a98347401,45367,1732018404555 2024-11-19T12:13:25,817 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740 2024-11-19T12:13:25,819 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740 2024-11-19T12:13:25,819 DEBUG [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:25,819 DEBUG [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36199 2024-11-19T12:13:25,819 DEBUG [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T12:13:25,822 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 351a98347401,35543,1732018404609 2024-11-19T12:13:25,822 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] master.ServerManager(517): Registering regionserver=351a98347401,35543,1732018404609 2024-11-19T12:13:25,822 DEBUG [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:25,823 DEBUG [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36199 2024-11-19T12:13:25,823 DEBUG [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T12:13:25,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T12:13:25,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T12:13:25,825 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T12:13:25,827 DEBUG [RS:2;351a98347401:35543 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:25,828 DEBUG [RS:2;351a98347401:35543 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36199 2024-11-19T12:13:25,828 DEBUG [RS:2;351a98347401:35543 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T12:13:25,829 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T12:13:25,834 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:25,835 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66849324, jitterRate=-0.0038674473762512207}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T12:13:25,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T12:13:25,840 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732018405789Initializing all the Stores at 1732018405793 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732018405793Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732018405793Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018405793Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732018405793Cleaning up temporary data from old regions at 1732018405823 (+30 ms)Region opened successfully at 1732018405839 (+16 ms) 2024-11-19T12:13:25,840 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T12:13:25,840 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T12:13:25,840 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T12:13:25,840 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T12:13:25,840 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T12:13:25,842 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T12:13:25,842 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732018405840Disabling compacts and flushes for region at 1732018405840Disabling writes for close at 1732018405840Writing region close event to WAL at 1732018405841 (+1 ms)Closed at 1732018405841 2024-11-19T12:13:25,844 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:25,845 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T12:13:25,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T12:13:25,862 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:25,865 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T12:13:25,880 DEBUG [RS:1;351a98347401:45367 {}] zookeeper.ZKUtil(111): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/351a98347401,45367,1732018404555 2024-11-19T12:13:25,880 WARN [RS:1;351a98347401:45367 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T12:13:25,880 DEBUG [RS:2;351a98347401:35543 {}] zookeeper.ZKUtil(111): regionserver:35543-0x1015303718b0003, quorum=127.0.0.1:57377, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/351a98347401,35543,1732018404609 2024-11-19T12:13:25,880 INFO [RS:1;351a98347401:45367 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T12:13:25,880 DEBUG [RS:0;351a98347401:36275 {}] zookeeper.ZKUtil(111): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/351a98347401,36275,1732018404438 2024-11-19T12:13:25,880 WARN [RS:2;351a98347401:35543 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T12:13:25,880 WARN [RS:0;351a98347401:36275 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T12:13:25,881 INFO [RS:2;351a98347401:35543 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T12:13:25,881 DEBUG [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,45367,1732018404555 2024-11-19T12:13:25,881 INFO [RS:0;351a98347401:36275 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T12:13:25,881 DEBUG [RS:2;351a98347401:35543 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609 2024-11-19T12:13:25,881 DEBUG [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,36275,1732018404438 2024-11-19T12:13:25,882 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [351a98347401,45367,1732018404555] 2024-11-19T12:13:25,882 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [351a98347401,35543,1732018404609] 2024-11-19T12:13:25,882 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [351a98347401,36275,1732018404438] 2024-11-19T12:13:25,913 INFO [RS:1;351a98347401:45367 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T12:13:25,913 INFO [RS:2;351a98347401:35543 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T12:13:25,913 INFO [RS:0;351a98347401:36275 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T12:13:25,930 INFO [RS:2;351a98347401:35543 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T12:13:25,930 INFO [RS:0;351a98347401:36275 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T12:13:25,930 INFO [RS:1;351a98347401:45367 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T12:13:25,936 INFO [RS:0;351a98347401:36275 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T12:13:25,936 INFO [RS:1;351a98347401:45367 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T12:13:25,936 INFO [RS:2;351a98347401:35543 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T12:13:25,936 INFO [RS:1;351a98347401:45367 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,936 INFO [RS:2;351a98347401:35543 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,936 INFO [RS:0;351a98347401:36275 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,937 INFO [RS:2;351a98347401:35543 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T12:13:25,937 INFO [RS:0;351a98347401:36275 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T12:13:25,938 INFO [RS:1;351a98347401:45367 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T12:13:25,943 INFO [RS:1;351a98347401:45367 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T12:13:25,943 INFO [RS:0;351a98347401:36275 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T12:13:25,943 INFO [RS:2;351a98347401:35543 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T12:13:25,945 INFO [RS:0;351a98347401:36275 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,945 INFO [RS:1;351a98347401:45367 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,945 INFO [RS:2;351a98347401:35543 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,945 DEBUG [RS:1;351a98347401:45367 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,945 DEBUG [RS:0;351a98347401:36275 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,945 DEBUG [RS:2;351a98347401:35543 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,945 DEBUG [RS:0;351a98347401:36275 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,945 DEBUG [RS:1;351a98347401:45367 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,945 DEBUG [RS:2;351a98347401:35543 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,945 DEBUG [RS:0;351a98347401:36275 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,945 DEBUG [RS:1;351a98347401:45367 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,945 DEBUG [RS:2;351a98347401:35543 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,945 DEBUG [RS:0;351a98347401:36275 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,945 DEBUG [RS:1;351a98347401:45367 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,945 DEBUG [RS:0;351a98347401:36275 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,946 DEBUG [RS:1;351a98347401:45367 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,946 DEBUG [RS:2;351a98347401:35543 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,946 DEBUG [RS:0;351a98347401:36275 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/351a98347401:0, corePoolSize=2, maxPoolSize=2 2024-11-19T12:13:25,946 DEBUG [RS:1;351a98347401:45367 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/351a98347401:0, corePoolSize=2, maxPoolSize=2 2024-11-19T12:13:25,946 DEBUG [RS:2;351a98347401:35543 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,946 DEBUG [RS:0;351a98347401:36275 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,946 DEBUG [RS:1;351a98347401:45367 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,946 DEBUG [RS:0;351a98347401:36275 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,946 DEBUG [RS:1;351a98347401:45367 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,946 DEBUG [RS:2;351a98347401:35543 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/351a98347401:0, corePoolSize=2, maxPoolSize=2 2024-11-19T12:13:25,946 DEBUG [RS:0;351a98347401:36275 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,946 DEBUG [RS:1;351a98347401:45367 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,946 DEBUG [RS:2;351a98347401:35543 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,946 DEBUG [RS:0;351a98347401:36275 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,946 DEBUG [RS:1;351a98347401:45367 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,946 DEBUG [RS:0;351a98347401:36275 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,946 DEBUG [RS:1;351a98347401:45367 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,946 DEBUG [RS:2;351a98347401:35543 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,946 DEBUG [RS:0;351a98347401:36275 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,946 DEBUG [RS:1;351a98347401:45367 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,946 DEBUG [RS:2;351a98347401:35543 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,946 DEBUG [RS:1;351a98347401:45367 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/351a98347401:0, corePoolSize=3, maxPoolSize=3 2024-11-19T12:13:25,946 DEBUG [RS:0;351a98347401:36275 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/351a98347401:0, corePoolSize=3, maxPoolSize=3 2024-11-19T12:13:25,947 DEBUG [RS:2;351a98347401:35543 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,947 DEBUG [RS:1;351a98347401:45367 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/351a98347401:0, corePoolSize=3, maxPoolSize=3 2024-11-19T12:13:25,947 DEBUG [RS:0;351a98347401:36275 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/351a98347401:0, corePoolSize=3, maxPoolSize=3 2024-11-19T12:13:25,947 DEBUG [RS:2;351a98347401:35543 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,947 DEBUG [RS:2;351a98347401:35543 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/351a98347401:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:13:25,947 DEBUG [RS:2;351a98347401:35543 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/351a98347401:0, corePoolSize=3, maxPoolSize=3 2024-11-19T12:13:25,947 DEBUG [RS:2;351a98347401:35543 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/351a98347401:0, corePoolSize=3, maxPoolSize=3 2024-11-19T12:13:25,952 INFO [RS:2;351a98347401:35543 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,952 INFO [RS:0;351a98347401:36275 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,952 INFO [RS:2;351a98347401:35543 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,952 INFO [RS:0;351a98347401:36275 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,952 INFO [RS:0;351a98347401:36275 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,952 INFO [RS:2;351a98347401:35543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,952 INFO [RS:0;351a98347401:36275 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,952 INFO [RS:2;351a98347401:35543 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,952 INFO [RS:0;351a98347401:36275 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,952 INFO [RS:2;351a98347401:35543 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,952 INFO [RS:0;351a98347401:36275 {}] hbase.ChoreService(168): Chore ScheduledChore name=351a98347401,36275,1732018404438-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T12:13:25,952 INFO [RS:2;351a98347401:35543 {}] hbase.ChoreService(168): Chore ScheduledChore name=351a98347401,35543,1732018404609-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T12:13:25,956 INFO [RS:1;351a98347401:45367 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,956 INFO [RS:1;351a98347401:45367 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,956 INFO [RS:1;351a98347401:45367 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,957 INFO [RS:1;351a98347401:45367 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,957 INFO [RS:1;351a98347401:45367 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,957 INFO [RS:1;351a98347401:45367 {}] hbase.ChoreService(168): Chore ScheduledChore name=351a98347401,45367,1732018404555-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T12:13:25,974 INFO [RS:2;351a98347401:35543 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T12:13:25,977 INFO [RS:2;351a98347401:35543 {}] hbase.ChoreService(168): Chore ScheduledChore name=351a98347401,35543,1732018404609-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,977 INFO [RS:2;351a98347401:35543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,977 INFO [RS:2;351a98347401:35543 {}] regionserver.Replication(171): 351a98347401,35543,1732018404609 started 2024-11-19T12:13:25,977 INFO [RS:1;351a98347401:45367 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T12:13:25,977 INFO [RS:0;351a98347401:36275 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T12:13:25,978 INFO [RS:0;351a98347401:36275 {}] hbase.ChoreService(168): Chore ScheduledChore name=351a98347401,36275,1732018404438-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,978 INFO [RS:1;351a98347401:45367 {}] hbase.ChoreService(168): Chore ScheduledChore name=351a98347401,45367,1732018404555-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,978 INFO [RS:0;351a98347401:36275 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,978 INFO [RS:0;351a98347401:36275 {}] regionserver.Replication(171): 351a98347401,36275,1732018404438 started 2024-11-19T12:13:25,978 INFO [RS:1;351a98347401:45367 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:25,978 INFO [RS:1;351a98347401:45367 {}] regionserver.Replication(171): 351a98347401,45367,1732018404555 started 2024-11-19T12:13:26,003 INFO [RS:0;351a98347401:36275 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:26,003 INFO [RS:2;351a98347401:35543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:26,003 INFO [RS:1;351a98347401:45367 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:26,003 INFO [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(1482): Serving as 351a98347401,36275,1732018404438, RpcServer on 351a98347401/172.17.0.2:36275, sessionid=0x1015303718b0001 2024-11-19T12:13:26,003 INFO [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(1482): Serving as 351a98347401,45367,1732018404555, RpcServer on 351a98347401/172.17.0.2:45367, sessionid=0x1015303718b0002 2024-11-19T12:13:26,003 INFO [RS:2;351a98347401:35543 {}] regionserver.HRegionServer(1482): Serving as 351a98347401,35543,1732018404609, RpcServer on 351a98347401/172.17.0.2:35543, sessionid=0x1015303718b0003 2024-11-19T12:13:26,004 DEBUG [RS:2;351a98347401:35543 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T12:13:26,004 DEBUG [RS:1;351a98347401:45367 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T12:13:26,004 DEBUG [RS:0;351a98347401:36275 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T12:13:26,004 DEBUG [RS:0;351a98347401:36275 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 351a98347401,36275,1732018404438 2024-11-19T12:13:26,004 DEBUG [RS:2;351a98347401:35543 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 351a98347401,35543,1732018404609 2024-11-19T12:13:26,004 DEBUG [RS:1;351a98347401:45367 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 351a98347401,45367,1732018404555 2024-11-19T12:13:26,005 DEBUG [RS:0;351a98347401:36275 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '351a98347401,36275,1732018404438' 2024-11-19T12:13:26,005 DEBUG [RS:2;351a98347401:35543 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '351a98347401,35543,1732018404609' 2024-11-19T12:13:26,005 DEBUG [RS:1;351a98347401:45367 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '351a98347401,45367,1732018404555' 2024-11-19T12:13:26,005 DEBUG [RS:2;351a98347401:35543 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T12:13:26,005 DEBUG [RS:0;351a98347401:36275 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T12:13:26,005 DEBUG [RS:1;351a98347401:45367 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T12:13:26,006 DEBUG [RS:2;351a98347401:35543 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T12:13:26,006 DEBUG [RS:0;351a98347401:36275 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T12:13:26,006 DEBUG [RS:1;351a98347401:45367 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T12:13:26,007 DEBUG [RS:2;351a98347401:35543 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T12:13:26,007 DEBUG [RS:0;351a98347401:36275 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T12:13:26,007 DEBUG [RS:1;351a98347401:45367 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T12:13:26,007 DEBUG [RS:2;351a98347401:35543 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T12:13:26,007 DEBUG [RS:0;351a98347401:36275 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T12:13:26,007 DEBUG [RS:1;351a98347401:45367 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T12:13:26,007 DEBUG [RS:2;351a98347401:35543 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 351a98347401,35543,1732018404609 2024-11-19T12:13:26,007 DEBUG [RS:1;351a98347401:45367 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 351a98347401,45367,1732018404555 2024-11-19T12:13:26,007 DEBUG [RS:0;351a98347401:36275 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 351a98347401,36275,1732018404438 2024-11-19T12:13:26,007 DEBUG [RS:0;351a98347401:36275 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '351a98347401,36275,1732018404438' 2024-11-19T12:13:26,007 DEBUG [RS:2;351a98347401:35543 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '351a98347401,35543,1732018404609' 2024-11-19T12:13:26,007 DEBUG [RS:1;351a98347401:45367 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '351a98347401,45367,1732018404555' 2024-11-19T12:13:26,007 DEBUG [RS:2;351a98347401:35543 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T12:13:26,007 DEBUG [RS:0;351a98347401:36275 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T12:13:26,007 DEBUG [RS:1;351a98347401:45367 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T12:13:26,008 DEBUG [RS:1;351a98347401:45367 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T12:13:26,008 DEBUG [RS:0;351a98347401:36275 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T12:13:26,008 DEBUG [RS:2;351a98347401:35543 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T12:13:26,008 DEBUG [RS:1;351a98347401:45367 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T12:13:26,009 INFO [RS:1;351a98347401:45367 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T12:13:26,009 DEBUG [RS:0;351a98347401:36275 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T12:13:26,009 INFO [RS:1;351a98347401:45367 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T12:13:26,009 INFO [RS:0;351a98347401:36275 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T12:13:26,009 DEBUG [RS:2;351a98347401:35543 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T12:13:26,009 INFO [RS:0;351a98347401:36275 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T12:13:26,009 INFO [RS:2;351a98347401:35543 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T12:13:26,009 INFO [RS:2;351a98347401:35543 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T12:13:26,016 WARN [351a98347401:38839 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-19T12:13:26,113 INFO [RS:2;351a98347401:35543 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T12:13:26,113 INFO [RS:0;351a98347401:36275 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T12:13:26,113 INFO [RS:1;351a98347401:45367 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T12:13:26,116 INFO [RS:2;351a98347401:35543 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=351a98347401%2C35543%2C1732018404609, suffix=, logDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609, archiveDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/oldWALs, maxLogs=32 2024-11-19T12:13:26,116 INFO [RS:1;351a98347401:45367 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=351a98347401%2C45367%2C1732018404555, suffix=, logDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,45367,1732018404555, archiveDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/oldWALs, maxLogs=32 2024-11-19T12:13:26,116 INFO [RS:0;351a98347401:36275 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=351a98347401%2C36275%2C1732018404438, suffix=, logDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,36275,1732018404438, archiveDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/oldWALs, maxLogs=32 2024-11-19T12:13:26,132 DEBUG [RS:2;351a98347401:35543 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609/351a98347401%2C35543%2C1732018404609.1732018406120, exclude list is [], retry=0 2024-11-19T12:13:26,132 DEBUG [RS:0;351a98347401:36275 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,36275,1732018404438/351a98347401%2C36275%2C1732018404438.1732018406120, exclude list is [], retry=0 2024-11-19T12:13:26,132 DEBUG [RS:1;351a98347401:45367 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,45367,1732018404555/351a98347401%2C45367%2C1732018404555.1732018406120, exclude list is [], retry=0 2024-11-19T12:13:26,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46231,DS-598e6c09-e04f-4361-a363-9b507e9888f8,DISK] 2024-11-19T12:13:26,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40579,DS-fc24d53f-eb0d-4d9b-b2d4-feb49c03511f,DISK] 2024-11-19T12:13:26,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40579,DS-fc24d53f-eb0d-4d9b-b2d4-feb49c03511f,DISK] 2024-11-19T12:13:26,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46857,DS-73e25d91-6f88-4582-919b-1060a325c9ba,DISK] 2024-11-19T12:13:26,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46857,DS-73e25d91-6f88-4582-919b-1060a325c9ba,DISK] 2024-11-19T12:13:26,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46231,DS-598e6c09-e04f-4361-a363-9b507e9888f8,DISK] 2024-11-19T12:13:26,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46231,DS-598e6c09-e04f-4361-a363-9b507e9888f8,DISK] 2024-11-19T12:13:26,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46857,DS-73e25d91-6f88-4582-919b-1060a325c9ba,DISK] 2024-11-19T12:13:26,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40579,DS-fc24d53f-eb0d-4d9b-b2d4-feb49c03511f,DISK] 2024-11-19T12:13:26,146 INFO [RS:0;351a98347401:36275 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,36275,1732018404438/351a98347401%2C36275%2C1732018404438.1732018406120 2024-11-19T12:13:26,148 DEBUG [RS:0;351a98347401:36275 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39493:39493),(127.0.0.1/127.0.0.1:45779:45779),(127.0.0.1/127.0.0.1:40699:40699)] 2024-11-19T12:13:26,150 INFO [RS:1;351a98347401:45367 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,45367,1732018404555/351a98347401%2C45367%2C1732018404555.1732018406120 2024-11-19T12:13:26,150 DEBUG [RS:1;351a98347401:45367 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40699:40699),(127.0.0.1/127.0.0.1:39493:39493),(127.0.0.1/127.0.0.1:45779:45779)] 2024-11-19T12:13:26,151 INFO [RS:2;351a98347401:35543 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609/351a98347401%2C35543%2C1732018404609.1732018406120 2024-11-19T12:13:26,151 DEBUG [RS:2;351a98347401:35543 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40699:40699),(127.0.0.1/127.0.0.1:39493:39493),(127.0.0.1/127.0.0.1:45779:45779)] 2024-11-19T12:13:26,272 DEBUG [351a98347401:38839 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-19T12:13:26,280 DEBUG [351a98347401:38839 {}] balancer.BalancerClusterState(204): Hosts are {351a98347401=0} racks are {/default-rack=0} 2024-11-19T12:13:26,285 DEBUG [351a98347401:38839 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T12:13:26,285 DEBUG [351a98347401:38839 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T12:13:26,285 DEBUG [351a98347401:38839 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-19T12:13:26,285 DEBUG [351a98347401:38839 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T12:13:26,286 DEBUG [351a98347401:38839 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T12:13:26,286 DEBUG [351a98347401:38839 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-19T12:13:26,286 INFO [351a98347401:38839 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T12:13:26,286 INFO [351a98347401:38839 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T12:13:26,286 INFO [351a98347401:38839 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-19T12:13:26,286 DEBUG [351a98347401:38839 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T12:13:26,292 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=351a98347401,35543,1732018404609 2024-11-19T12:13:26,298 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,35543,1732018404609, state=OPENING 2024-11-19T12:13:26,353 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T12:13:26,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:26,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:26,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35543-0x1015303718b0003, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:26,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:26,365 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:26,365 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:26,365 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:26,365 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:26,366 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:26,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,35543,1732018404609}] 2024-11-19T12:13:26,545 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T12:13:26,547 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54803, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T12:13:26,560 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T12:13:26,561 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T12:13:26,561 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T12:13:26,565 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=351a98347401%2C35543%2C1732018404609.meta, suffix=.meta, logDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609, archiveDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/oldWALs, maxLogs=32 2024-11-19T12:13:26,581 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609/351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, exclude list is [], retry=0 2024-11-19T12:13:26,585 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46857,DS-73e25d91-6f88-4582-919b-1060a325c9ba,DISK] 2024-11-19T12:13:26,585 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46231,DS-598e6c09-e04f-4361-a363-9b507e9888f8,DISK] 2024-11-19T12:13:26,585 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40579,DS-fc24d53f-eb0d-4d9b-b2d4-feb49c03511f,DISK] 2024-11-19T12:13:26,588 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609/351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta 2024-11-19T12:13:26,588 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40699:40699),(127.0.0.1/127.0.0.1:45779:45779),(127.0.0.1/127.0.0.1:39493:39493)] 2024-11-19T12:13:26,589 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:13:26,591 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T12:13:26,594 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T12:13:26,599 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T12:13:26,602 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T12:13:26,603 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:26,603 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T12:13:26,603 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T12:13:26,607 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T12:13:26,609 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T12:13:26,609 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:26,610 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:13:26,610 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T12:13:26,612 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T12:13:26,612 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:26,612 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:13:26,613 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T12:13:26,614 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T12:13:26,614 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:26,615 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:13:26,615 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T12:13:26,617 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T12:13:26,617 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:26,618 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:13:26,618 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T12:13:26,620 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740 2024-11-19T12:13:26,623 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740 2024-11-19T12:13:26,627 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T12:13:26,627 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T12:13:26,628 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T12:13:26,631 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T12:13:26,633 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63937533, jitterRate=-0.04725651443004608}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T12:13:26,634 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T12:13:26,635 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732018406604Writing region info on filesystem at 1732018406604Initializing all the Stores at 1732018406606 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732018406606Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732018406607 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018406607Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732018406607Cleaning up temporary data from old regions at 1732018406627 (+20 ms)Running coprocessor post-open hooks at 1732018406634 (+7 ms)Region opened successfully at 1732018406635 (+1 ms) 2024-11-19T12:13:26,642 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732018406537 2024-11-19T12:13:26,653 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T12:13:26,653 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T12:13:26,655 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,35543,1732018404609 2024-11-19T12:13:26,658 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,35543,1732018404609, state=OPEN 2024-11-19T12:13:26,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:26,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:26,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:26,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:26,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:26,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:26,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35543-0x1015303718b0003, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:26,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:26,670 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=351a98347401,35543,1732018404609 2024-11-19T12:13:26,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T12:13:26,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,35543,1732018404609 in 302 msec 2024-11-19T12:13:26,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T12:13:26,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 827 msec 2024-11-19T12:13:26,686 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:26,686 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T12:13:26,707 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:13:26,709 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=351a98347401,35543,1732018404609, seqNum=-1] 2024-11-19T12:13:26,730 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:13:26,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44951, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:13:26,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1280 sec 2024-11-19T12:13:26,755 INFO [master/351a98347401:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732018406755, completionTime=-1 2024-11-19T12:13:26,758 INFO [master/351a98347401:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-19T12:13:26,758 DEBUG [master/351a98347401:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:26,786 INFO [master/351a98347401:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-19T12:13:26,786 INFO [master/351a98347401:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018466786 2024-11-19T12:13:26,786 INFO [master/351a98347401:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018526786 2024-11-19T12:13:26,786 INFO [master/351a98347401:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 28 msec 2024-11-19T12:13:26,788 DEBUG [master/351a98347401:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:26,796 INFO [master/351a98347401:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=351a98347401,38839,1732018403620-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:26,796 INFO [master/351a98347401:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=351a98347401,38839,1732018403620-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:26,797 INFO [master/351a98347401:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=351a98347401,38839,1732018403620-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:26,798 INFO [master/351a98347401:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-351a98347401:38839, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:26,799 INFO [master/351a98347401:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:26,799 INFO [master/351a98347401:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:26,805 DEBUG [master/351a98347401:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T12:13:26,828 INFO [master/351a98347401:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.105sec 2024-11-19T12:13:26,830 INFO [master/351a98347401:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T12:13:26,831 INFO [master/351a98347401:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T12:13:26,831 INFO [master/351a98347401:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T12:13:26,832 INFO [master/351a98347401:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T12:13:26,832 INFO [master/351a98347401:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T12:13:26,832 INFO [master/351a98347401:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=351a98347401,38839,1732018403620-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T12:13:26,833 INFO [master/351a98347401:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=351a98347401,38839,1732018403620-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T12:13:26,837 DEBUG [master/351a98347401:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T12:13:26,838 INFO [master/351a98347401:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T12:13:26,838 INFO [master/351a98347401:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=351a98347401,38839,1732018403620-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:13:26,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3475a1ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:13:26,870 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-19T12:13:26,870 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-19T12:13:26,874 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 351a98347401,38839,-1 for getting cluster id 2024-11-19T12:13:26,876 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:13:26,884 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '065e031d-65df-4d23-b424-83fdc49a195f' 2024-11-19T12:13:26,886 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:13:26,886 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "065e031d-65df-4d23-b424-83fdc49a195f" 2024-11-19T12:13:26,887 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34831515, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:13:26,887 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [351a98347401,38839,-1] 2024-11-19T12:13:26,889 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:13:26,891 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:13:26,892 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:13:26,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e978e9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:13:26,895 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:13:26,903 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=351a98347401,35543,1732018404609, seqNum=-1] 2024-11-19T12:13:26,903 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:13:26,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:13:26,926 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=351a98347401,38839,1732018403620 2024-11-19T12:13:26,930 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T12:13:26,953 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 351a98347401,38839,1732018403620 2024-11-19T12:13:26,955 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@12daeb33 2024-11-19T12:13:26,956 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T12:13:26,959 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T12:13:26,967 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T12:13:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:26,986 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T12:13:26,988 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-19T12:13:26,989 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:26,991 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T12:13:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T12:13:27,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741837_1013 (size=358) 2024-11-19T12:13:27,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741837_1013 (size=358) 2024-11-19T12:13:27,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741837_1013 (size=358) 2024-11-19T12:13:27,010 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b859464d4ffda6f2df23654fb949364f, NAME => 'test,,1732018406961.b859464d4ffda6f2df23654fb949364f.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,010 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 5b9b9ba699dcd89ef4b3398f2beb2455, NAME => 'test,ccc,1732018406961.5b9b9ba699dcd89ef4b3398f2beb2455.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,011 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => e257775d8750f12b70161bb0b6387163, NAME => 'test,bbb,1732018406961.e257775d8750f12b70161bb0b6387163.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741838_1014 (size=45) 2024-11-19T12:13:27,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741838_1014 (size=45) 2024-11-19T12:13:27,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741838_1014 (size=45) 2024-11-19T12:13:27,034 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1732018406961.5b9b9ba699dcd89ef4b3398f2beb2455.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,034 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 5b9b9ba699dcd89ef4b3398f2beb2455, disabling compactions & flushes 2024-11-19T12:13:27,034 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1732018406961.5b9b9ba699dcd89ef4b3398f2beb2455. 2024-11-19T12:13:27,034 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732018406961.5b9b9ba699dcd89ef4b3398f2beb2455. 2024-11-19T12:13:27,034 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732018406961.5b9b9ba699dcd89ef4b3398f2beb2455. after waiting 0 ms 2024-11-19T12:13:27,034 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732018406961.5b9b9ba699dcd89ef4b3398f2beb2455. 2024-11-19T12:13:27,034 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1732018406961.5b9b9ba699dcd89ef4b3398f2beb2455. 2024-11-19T12:13:27,034 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 5b9b9ba699dcd89ef4b3398f2beb2455: Waiting for close lock at 1732018407034Disabling compacts and flushes for region at 1732018407034Disabling writes for close at 1732018407034Writing region close event to WAL at 1732018407034Closed at 1732018407034 2024-11-19T12:13:27,035 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 8fbade10b0e1d2be0489df2d6c3fcf9d, NAME => 'test,ddd,1732018406961.8fbade10b0e1d2be0489df2d6c3fcf9d.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741839_1015 (size=45) 2024-11-19T12:13:27,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741840_1016 (size=42) 2024-11-19T12:13:27,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741839_1015 (size=45) 2024-11-19T12:13:27,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741840_1016 (size=42) 2024-11-19T12:13:27,046 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1732018406961.b859464d4ffda6f2df23654fb949364f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741840_1016 (size=42) 2024-11-19T12:13:27,046 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing b859464d4ffda6f2df23654fb949364f, disabling compactions & flushes 2024-11-19T12:13:27,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741839_1015 (size=45) 2024-11-19T12:13:27,046 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1732018406961.b859464d4ffda6f2df23654fb949364f. 2024-11-19T12:13:27,046 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732018406961.b859464d4ffda6f2df23654fb949364f. 2024-11-19T12:13:27,047 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1732018406961.b859464d4ffda6f2df23654fb949364f. after waiting 0 ms 2024-11-19T12:13:27,047 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1732018406961.b859464d4ffda6f2df23654fb949364f. 2024-11-19T12:13:27,047 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1732018406961.b859464d4ffda6f2df23654fb949364f. 2024-11-19T12:13:27,047 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for b859464d4ffda6f2df23654fb949364f: Waiting for close lock at 1732018407046Disabling compacts and flushes for region at 1732018407046Disabling writes for close at 1732018407047 (+1 ms)Writing region close event to WAL at 1732018407047Closed at 1732018407047 2024-11-19T12:13:27,047 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f9224ed53fa7acfead92d4143c8626ff, NAME => 'test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,048 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1732018406961.e257775d8750f12b70161bb0b6387163.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,049 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing e257775d8750f12b70161bb0b6387163, disabling compactions & flushes 2024-11-19T12:13:27,049 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1732018406961.e257775d8750f12b70161bb0b6387163. 2024-11-19T12:13:27,049 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732018406961.e257775d8750f12b70161bb0b6387163. 2024-11-19T12:13:27,049 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732018406961.e257775d8750f12b70161bb0b6387163. after waiting 0 ms 2024-11-19T12:13:27,049 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732018406961.e257775d8750f12b70161bb0b6387163. 2024-11-19T12:13:27,049 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1732018406961.e257775d8750f12b70161bb0b6387163. 2024-11-19T12:13:27,049 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for e257775d8750f12b70161bb0b6387163: Waiting for close lock at 1732018407049Disabling compacts and flushes for region at 1732018407049Disabling writes for close at 1732018407049Writing region close event to WAL at 1732018407049Closed at 1732018407049 2024-11-19T12:13:27,050 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => a78d1de57ec917279b938f869bdbe10d, NAME => 'test,fff,1732018406961.a78d1de57ec917279b938f869bdbe10d.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741841_1017 (size=45) 2024-11-19T12:13:27,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741841_1017 (size=45) 2024-11-19T12:13:27,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741841_1017 (size=45) 2024-11-19T12:13:27,066 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ddd,1732018406961.8fbade10b0e1d2be0489df2d6c3fcf9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,066 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 8fbade10b0e1d2be0489df2d6c3fcf9d, disabling compactions & flushes 2024-11-19T12:13:27,066 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ddd,1732018406961.8fbade10b0e1d2be0489df2d6c3fcf9d. 2024-11-19T12:13:27,066 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732018406961.8fbade10b0e1d2be0489df2d6c3fcf9d. 2024-11-19T12:13:27,066 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732018406961.8fbade10b0e1d2be0489df2d6c3fcf9d. after waiting 0 ms 2024-11-19T12:13:27,066 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732018406961.8fbade10b0e1d2be0489df2d6c3fcf9d. 2024-11-19T12:13:27,066 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ddd,1732018406961.8fbade10b0e1d2be0489df2d6c3fcf9d. 2024-11-19T12:13:27,066 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 8fbade10b0e1d2be0489df2d6c3fcf9d: Waiting for close lock at 1732018407066Disabling compacts and flushes for region at 1732018407066Disabling writes for close at 1732018407066Writing region close event to WAL at 1732018407066Closed at 1732018407066 2024-11-19T12:13:27,067 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 125c93438c95e8e5f1b8be66c59d7cf9, NAME => 'test,ggg,1732018406961.125c93438c95e8e5f1b8be66c59d7cf9.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741842_1018 (size=45) 2024-11-19T12:13:27,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741842_1018 (size=45) 2024-11-19T12:13:27,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741842_1018 (size=45) 2024-11-19T12:13:27,082 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,082 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing f9224ed53fa7acfead92d4143c8626ff, disabling compactions & flushes 2024-11-19T12:13:27,082 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff. 2024-11-19T12:13:27,082 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff. 2024-11-19T12:13:27,082 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff. after waiting 0 ms 2024-11-19T12:13:27,082 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff. 2024-11-19T12:13:27,083 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff. 2024-11-19T12:13:27,083 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for f9224ed53fa7acfead92d4143c8626ff: Waiting for close lock at 1732018407082Disabling compacts and flushes for region at 1732018407082Disabling writes for close at 1732018407082Writing region close event to WAL at 1732018407083 (+1 ms)Closed at 1732018407083 2024-11-19T12:13:27,084 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7382ad40044282a358da8130ce023367, NAME => 'test,hhh,1732018406961.7382ad40044282a358da8130ce023367.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741843_1019 (size=45) 2024-11-19T12:13:27,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741843_1019 (size=45) 2024-11-19T12:13:27,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741843_1019 (size=45) 2024-11-19T12:13:27,099 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,fff,1732018406961.a78d1de57ec917279b938f869bdbe10d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,099 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing a78d1de57ec917279b938f869bdbe10d, disabling compactions & flushes 2024-11-19T12:13:27,099 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,fff,1732018406961.a78d1de57ec917279b938f869bdbe10d. 2024-11-19T12:13:27,099 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732018406961.a78d1de57ec917279b938f869bdbe10d. 2024-11-19T12:13:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T12:13:27,099 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732018406961.a78d1de57ec917279b938f869bdbe10d. after waiting 0 ms 2024-11-19T12:13:27,099 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732018406961.a78d1de57ec917279b938f869bdbe10d. 2024-11-19T12:13:27,099 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,fff,1732018406961.a78d1de57ec917279b938f869bdbe10d. 2024-11-19T12:13:27,099 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for a78d1de57ec917279b938f869bdbe10d: Waiting for close lock at 1732018407099Disabling compacts and flushes for region at 1732018407099Disabling writes for close at 1732018407099Writing region close event to WAL at 1732018407099Closed at 1732018407099 2024-11-19T12:13:27,100 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ab050043367e9fb30539590d5f965d47, NAME => 'test,iii,1732018406961.ab050043367e9fb30539590d5f965d47.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741844_1020 (size=45) 2024-11-19T12:13:27,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741844_1020 (size=45) 2024-11-19T12:13:27,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741844_1020 (size=45) 2024-11-19T12:13:27,107 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ggg,1732018406961.125c93438c95e8e5f1b8be66c59d7cf9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,107 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 125c93438c95e8e5f1b8be66c59d7cf9, disabling compactions & flushes 2024-11-19T12:13:27,107 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ggg,1732018406961.125c93438c95e8e5f1b8be66c59d7cf9. 2024-11-19T12:13:27,107 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732018406961.125c93438c95e8e5f1b8be66c59d7cf9. 2024-11-19T12:13:27,107 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732018406961.125c93438c95e8e5f1b8be66c59d7cf9. after waiting 0 ms 2024-11-19T12:13:27,107 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732018406961.125c93438c95e8e5f1b8be66c59d7cf9. 2024-11-19T12:13:27,108 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ggg,1732018406961.125c93438c95e8e5f1b8be66c59d7cf9. 2024-11-19T12:13:27,108 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 125c93438c95e8e5f1b8be66c59d7cf9: Waiting for close lock at 1732018407107Disabling compacts and flushes for region at 1732018407107Disabling writes for close at 1732018407107Writing region close event to WAL at 1732018407107Closed at 1732018407108 (+1 ms) 2024-11-19T12:13:27,109 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 19add541d34392b48fdd488f1c19c182, NAME => 'test,jjj,1732018406961.19add541d34392b48fdd488f1c19c182.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741845_1021 (size=45) 2024-11-19T12:13:27,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741845_1021 (size=45) 2024-11-19T12:13:27,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741845_1021 (size=45) 2024-11-19T12:13:27,122 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,hhh,1732018406961.7382ad40044282a358da8130ce023367.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,122 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 7382ad40044282a358da8130ce023367, disabling compactions & flushes 2024-11-19T12:13:27,122 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,hhh,1732018406961.7382ad40044282a358da8130ce023367. 2024-11-19T12:13:27,122 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732018406961.7382ad40044282a358da8130ce023367. 2024-11-19T12:13:27,123 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732018406961.7382ad40044282a358da8130ce023367. after waiting 0 ms 2024-11-19T12:13:27,123 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732018406961.7382ad40044282a358da8130ce023367. 2024-11-19T12:13:27,123 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,hhh,1732018406961.7382ad40044282a358da8130ce023367. 2024-11-19T12:13:27,123 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7382ad40044282a358da8130ce023367: Waiting for close lock at 1732018407122Disabling compacts and flushes for region at 1732018407122Disabling writes for close at 1732018407123 (+1 ms)Writing region close event to WAL at 1732018407123Closed at 1732018407123 2024-11-19T12:13:27,124 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a329354a61adc5325fa166bf564b84eb, NAME => 'test,kkk,1732018406961.a329354a61adc5325fa166bf564b84eb.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741846_1022 (size=45) 2024-11-19T12:13:27,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741846_1022 (size=45) 2024-11-19T12:13:27,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741846_1022 (size=45) 2024-11-19T12:13:27,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741847_1023 (size=45) 2024-11-19T12:13:27,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741847_1023 (size=45) 2024-11-19T12:13:27,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741847_1023 (size=45) 2024-11-19T12:13:27,164 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,jjj,1732018406961.19add541d34392b48fdd488f1c19c182.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,165 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 19add541d34392b48fdd488f1c19c182, disabling compactions & flushes 2024-11-19T12:13:27,165 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,jjj,1732018406961.19add541d34392b48fdd488f1c19c182. 2024-11-19T12:13:27,165 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732018406961.19add541d34392b48fdd488f1c19c182. 2024-11-19T12:13:27,165 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732018406961.19add541d34392b48fdd488f1c19c182. after waiting 0 ms 2024-11-19T12:13:27,165 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732018406961.19add541d34392b48fdd488f1c19c182. 2024-11-19T12:13:27,165 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,jjj,1732018406961.19add541d34392b48fdd488f1c19c182. 2024-11-19T12:13:27,165 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 19add541d34392b48fdd488f1c19c182: Waiting for close lock at 1732018407165Disabling compacts and flushes for region at 1732018407165Disabling writes for close at 1732018407165Writing region close event to WAL at 1732018407165Closed at 1732018407165 2024-11-19T12:13:27,166 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 5bc8ba883b6bd714e90daec2cb61209e, NAME => 'test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741848_1024 (size=45) 2024-11-19T12:13:27,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741848_1024 (size=45) 2024-11-19T12:13:27,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741848_1024 (size=45) 2024-11-19T12:13:27,179 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,kkk,1732018406961.a329354a61adc5325fa166bf564b84eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,179 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing a329354a61adc5325fa166bf564b84eb, disabling compactions & flushes 2024-11-19T12:13:27,179 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,kkk,1732018406961.a329354a61adc5325fa166bf564b84eb. 2024-11-19T12:13:27,179 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732018406961.a329354a61adc5325fa166bf564b84eb. 2024-11-19T12:13:27,179 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732018406961.a329354a61adc5325fa166bf564b84eb. after waiting 0 ms 2024-11-19T12:13:27,179 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732018406961.a329354a61adc5325fa166bf564b84eb. 2024-11-19T12:13:27,179 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,kkk,1732018406961.a329354a61adc5325fa166bf564b84eb. 2024-11-19T12:13:27,179 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for a329354a61adc5325fa166bf564b84eb: Waiting for close lock at 1732018407179Disabling compacts and flushes for region at 1732018407179Disabling writes for close at 1732018407179Writing region close event to WAL at 1732018407179Closed at 1732018407179 2024-11-19T12:13:27,180 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => adc3abaf30d887a0d57a7730313a62b4, NAME => 'test,mmm,1732018406961.adc3abaf30d887a0d57a7730313a62b4.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741849_1025 (size=45) 2024-11-19T12:13:27,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741849_1025 (size=45) 2024-11-19T12:13:27,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741849_1025 (size=45) 2024-11-19T12:13:27,194 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,194 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 5bc8ba883b6bd714e90daec2cb61209e, disabling compactions & flushes 2024-11-19T12:13:27,195 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e. 2024-11-19T12:13:27,195 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e. 2024-11-19T12:13:27,195 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e. after waiting 0 ms 2024-11-19T12:13:27,195 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e. 2024-11-19T12:13:27,195 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e. 2024-11-19T12:13:27,195 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 5bc8ba883b6bd714e90daec2cb61209e: Waiting for close lock at 1732018407194Disabling compacts and flushes for region at 1732018407194Disabling writes for close at 1732018407195 (+1 ms)Writing region close event to WAL at 1732018407195Closed at 1732018407195 2024-11-19T12:13:27,196 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => ef4298fd6621c8e7facf42422bd80be4, NAME => 'test,nnn,1732018406961.ef4298fd6621c8e7facf42422bd80be4.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741850_1026 (size=45) 2024-11-19T12:13:27,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741850_1026 (size=45) 2024-11-19T12:13:27,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741850_1026 (size=45) 2024-11-19T12:13:27,204 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,mmm,1732018406961.adc3abaf30d887a0d57a7730313a62b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,204 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing adc3abaf30d887a0d57a7730313a62b4, disabling compactions & flushes 2024-11-19T12:13:27,204 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,mmm,1732018406961.adc3abaf30d887a0d57a7730313a62b4. 2024-11-19T12:13:27,204 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732018406961.adc3abaf30d887a0d57a7730313a62b4. 2024-11-19T12:13:27,204 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732018406961.adc3abaf30d887a0d57a7730313a62b4. after waiting 0 ms 2024-11-19T12:13:27,204 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732018406961.adc3abaf30d887a0d57a7730313a62b4. 2024-11-19T12:13:27,204 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,mmm,1732018406961.adc3abaf30d887a0d57a7730313a62b4. 2024-11-19T12:13:27,205 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for adc3abaf30d887a0d57a7730313a62b4: Waiting for close lock at 1732018407204Disabling compacts and flushes for region at 1732018407204Disabling writes for close at 1732018407204Writing region close event to WAL at 1732018407204Closed at 1732018407204 2024-11-19T12:13:27,205 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d8bfccd6e991fb7a90ce5bcadc0a3ef5, NAME => 'test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741851_1027 (size=45) 2024-11-19T12:13:27,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741851_1027 (size=45) 2024-11-19T12:13:27,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741851_1027 (size=45) 2024-11-19T12:13:27,221 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,nnn,1732018406961.ef4298fd6621c8e7facf42422bd80be4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,221 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing ef4298fd6621c8e7facf42422bd80be4, disabling compactions & flushes 2024-11-19T12:13:27,221 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,nnn,1732018406961.ef4298fd6621c8e7facf42422bd80be4. 2024-11-19T12:13:27,221 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732018406961.ef4298fd6621c8e7facf42422bd80be4. 2024-11-19T12:13:27,221 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732018406961.ef4298fd6621c8e7facf42422bd80be4. after waiting 0 ms 2024-11-19T12:13:27,221 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732018406961.ef4298fd6621c8e7facf42422bd80be4. 2024-11-19T12:13:27,221 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,nnn,1732018406961.ef4298fd6621c8e7facf42422bd80be4. 2024-11-19T12:13:27,221 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for ef4298fd6621c8e7facf42422bd80be4: Waiting for close lock at 1732018407221Disabling compacts and flushes for region at 1732018407221Disabling writes for close at 1732018407221Writing region close event to WAL at 1732018407221Closed at 1732018407221 2024-11-19T12:13:27,222 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 963c42e1179165051d86c1560642c124, NAME => 'test,ppp,1732018406961.963c42e1179165051d86c1560642c124.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741852_1028 (size=45) 2024-11-19T12:13:27,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741852_1028 (size=45) 2024-11-19T12:13:27,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741852_1028 (size=45) 2024-11-19T12:13:27,236 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,236 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing d8bfccd6e991fb7a90ce5bcadc0a3ef5, disabling compactions & flushes 2024-11-19T12:13:27,236 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5. 2024-11-19T12:13:27,236 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5. 2024-11-19T12:13:27,237 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5. after waiting 1 ms 2024-11-19T12:13:27,237 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5. 2024-11-19T12:13:27,237 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5. 2024-11-19T12:13:27,237 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for d8bfccd6e991fb7a90ce5bcadc0a3ef5: Waiting for close lock at 1732018407236Disabling compacts and flushes for region at 1732018407236Disabling writes for close at 1732018407237 (+1 ms)Writing region close event to WAL at 1732018407237Closed at 1732018407237 2024-11-19T12:13:27,238 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ca6b3c7ec060f344fff683e91e796d33, NAME => 'test,qqq,1732018406961.ca6b3c7ec060f344fff683e91e796d33.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741853_1029 (size=45) 2024-11-19T12:13:27,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741853_1029 (size=45) 2024-11-19T12:13:27,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741853_1029 (size=45) 2024-11-19T12:13:27,250 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ppp,1732018406961.963c42e1179165051d86c1560642c124.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,250 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 963c42e1179165051d86c1560642c124, disabling compactions & flushes 2024-11-19T12:13:27,250 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ppp,1732018406961.963c42e1179165051d86c1560642c124. 2024-11-19T12:13:27,251 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732018406961.963c42e1179165051d86c1560642c124. 2024-11-19T12:13:27,251 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732018406961.963c42e1179165051d86c1560642c124. after waiting 0 ms 2024-11-19T12:13:27,251 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732018406961.963c42e1179165051d86c1560642c124. 2024-11-19T12:13:27,251 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ppp,1732018406961.963c42e1179165051d86c1560642c124. 2024-11-19T12:13:27,251 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 963c42e1179165051d86c1560642c124: Waiting for close lock at 1732018407250Disabling compacts and flushes for region at 1732018407250Disabling writes for close at 1732018407251 (+1 ms)Writing region close event to WAL at 1732018407251Closed at 1732018407251 2024-11-19T12:13:27,252 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 24b360a2b049dbc9c784cdd687321115, NAME => 'test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741854_1030 (size=45) 2024-11-19T12:13:27,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741854_1030 (size=45) 2024-11-19T12:13:27,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741854_1030 (size=45) 2024-11-19T12:13:27,263 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,qqq,1732018406961.ca6b3c7ec060f344fff683e91e796d33.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,263 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing ca6b3c7ec060f344fff683e91e796d33, disabling compactions & flushes 2024-11-19T12:13:27,263 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,qqq,1732018406961.ca6b3c7ec060f344fff683e91e796d33. 2024-11-19T12:13:27,263 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732018406961.ca6b3c7ec060f344fff683e91e796d33. 2024-11-19T12:13:27,263 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732018406961.ca6b3c7ec060f344fff683e91e796d33. after waiting 0 ms 2024-11-19T12:13:27,264 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732018406961.ca6b3c7ec060f344fff683e91e796d33. 2024-11-19T12:13:27,264 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,qqq,1732018406961.ca6b3c7ec060f344fff683e91e796d33. 2024-11-19T12:13:27,264 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for ca6b3c7ec060f344fff683e91e796d33: Waiting for close lock at 1732018407263Disabling compacts and flushes for region at 1732018407263Disabling writes for close at 1732018407263Writing region close event to WAL at 1732018407264 (+1 ms)Closed at 1732018407264 2024-11-19T12:13:27,265 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 797a72fe833580fc974f887653d7763a, NAME => 'test,sss,1732018406961.797a72fe833580fc974f887653d7763a.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741855_1031 (size=45) 2024-11-19T12:13:27,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741855_1031 (size=45) 2024-11-19T12:13:27,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741855_1031 (size=45) 2024-11-19T12:13:27,276 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,276 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 24b360a2b049dbc9c784cdd687321115, disabling compactions & flushes 2024-11-19T12:13:27,276 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115. 2024-11-19T12:13:27,277 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115. 2024-11-19T12:13:27,277 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115. after waiting 0 ms 2024-11-19T12:13:27,277 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115. 2024-11-19T12:13:27,277 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115. 2024-11-19T12:13:27,277 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 24b360a2b049dbc9c784cdd687321115: Waiting for close lock at 1732018407276Disabling compacts and flushes for region at 1732018407276Disabling writes for close at 1732018407277 (+1 ms)Writing region close event to WAL at 1732018407277Closed at 1732018407277 2024-11-19T12:13:27,278 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 2ec97ffd6b2cc998be397eec7fc0a0bc, NAME => 'test,ttt,1732018406961.2ec97ffd6b2cc998be397eec7fc0a0bc.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741856_1032 (size=45) 2024-11-19T12:13:27,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741856_1032 (size=45) 2024-11-19T12:13:27,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741856_1032 (size=45) 2024-11-19T12:13:27,285 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,sss,1732018406961.797a72fe833580fc974f887653d7763a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,285 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 797a72fe833580fc974f887653d7763a, disabling compactions & flushes 2024-11-19T12:13:27,285 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,sss,1732018406961.797a72fe833580fc974f887653d7763a. 2024-11-19T12:13:27,285 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732018406961.797a72fe833580fc974f887653d7763a. 2024-11-19T12:13:27,285 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732018406961.797a72fe833580fc974f887653d7763a. after waiting 0 ms 2024-11-19T12:13:27,285 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732018406961.797a72fe833580fc974f887653d7763a. 2024-11-19T12:13:27,286 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,sss,1732018406961.797a72fe833580fc974f887653d7763a. 2024-11-19T12:13:27,286 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 797a72fe833580fc974f887653d7763a: Waiting for close lock at 1732018407285Disabling compacts and flushes for region at 1732018407285Disabling writes for close at 1732018407285Writing region close event to WAL at 1732018407285Closed at 1732018407285 2024-11-19T12:13:27,286 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, NAME => 'test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741857_1033 (size=45) 2024-11-19T12:13:27,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741857_1033 (size=45) 2024-11-19T12:13:27,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741857_1033 (size=45) 2024-11-19T12:13:27,296 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ttt,1732018406961.2ec97ffd6b2cc998be397eec7fc0a0bc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,296 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 2ec97ffd6b2cc998be397eec7fc0a0bc, disabling compactions & flushes 2024-11-19T12:13:27,297 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ttt,1732018406961.2ec97ffd6b2cc998be397eec7fc0a0bc. 2024-11-19T12:13:27,297 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732018406961.2ec97ffd6b2cc998be397eec7fc0a0bc. 2024-11-19T12:13:27,297 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732018406961.2ec97ffd6b2cc998be397eec7fc0a0bc. after waiting 0 ms 2024-11-19T12:13:27,297 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732018406961.2ec97ffd6b2cc998be397eec7fc0a0bc. 2024-11-19T12:13:27,297 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ttt,1732018406961.2ec97ffd6b2cc998be397eec7fc0a0bc. 2024-11-19T12:13:27,297 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 2ec97ffd6b2cc998be397eec7fc0a0bc: Waiting for close lock at 1732018407296Disabling compacts and flushes for region at 1732018407296Disabling writes for close at 1732018407297 (+1 ms)Writing region close event to WAL at 1732018407297Closed at 1732018407297 2024-11-19T12:13:27,298 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 2b6d3f4b598ab015d9c62b67f0bac50f, NAME => 'test,vvv,1732018406961.2b6d3f4b598ab015d9c62b67f0bac50f.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741858_1034 (size=45) 2024-11-19T12:13:27,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741858_1034 (size=45) 2024-11-19T12:13:27,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741858_1034 (size=45) 2024-11-19T12:13:27,307 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,307 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, disabling compactions & flushes 2024-11-19T12:13:27,307 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3. 2024-11-19T12:13:27,308 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3. 2024-11-19T12:13:27,308 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3. after waiting 0 ms 2024-11-19T12:13:27,308 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3. 2024-11-19T12:13:27,308 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3. 2024-11-19T12:13:27,308 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for c48a9e94bd9fcbd6f8c1f6ae2b6c80d3: Waiting for close lock at 1732018407307Disabling compacts and flushes for region at 1732018407307Disabling writes for close at 1732018407308 (+1 ms)Writing region close event to WAL at 1732018407308Closed at 1732018407308 2024-11-19T12:13:27,309 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e85bee36dde7095f8d08f860a32613d7, NAME => 'test,www,1732018406961.e85bee36dde7095f8d08f860a32613d7.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T12:13:27,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741859_1035 (size=45) 2024-11-19T12:13:27,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741859_1035 (size=45) 2024-11-19T12:13:27,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741859_1035 (size=45) 2024-11-19T12:13:27,314 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,vvv,1732018406961.2b6d3f4b598ab015d9c62b67f0bac50f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,314 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 2b6d3f4b598ab015d9c62b67f0bac50f, disabling compactions & flushes 2024-11-19T12:13:27,314 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,vvv,1732018406961.2b6d3f4b598ab015d9c62b67f0bac50f. 2024-11-19T12:13:27,314 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732018406961.2b6d3f4b598ab015d9c62b67f0bac50f. 2024-11-19T12:13:27,314 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732018406961.2b6d3f4b598ab015d9c62b67f0bac50f. after waiting 0 ms 2024-11-19T12:13:27,314 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732018406961.2b6d3f4b598ab015d9c62b67f0bac50f. 2024-11-19T12:13:27,314 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,vvv,1732018406961.2b6d3f4b598ab015d9c62b67f0bac50f. 2024-11-19T12:13:27,314 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 2b6d3f4b598ab015d9c62b67f0bac50f: Waiting for close lock at 1732018407314Disabling compacts and flushes for region at 1732018407314Disabling writes for close at 1732018407314Writing region close event to WAL at 1732018407314Closed at 1732018407314 2024-11-19T12:13:27,315 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 11d894b29c3ee7cc1e5281c570908571, NAME => 'test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741860_1036 (size=45) 2024-11-19T12:13:27,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741860_1036 (size=45) 2024-11-19T12:13:27,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741860_1036 (size=45) 2024-11-19T12:13:27,321 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,www,1732018406961.e85bee36dde7095f8d08f860a32613d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,321 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing e85bee36dde7095f8d08f860a32613d7, disabling compactions & flushes 2024-11-19T12:13:27,321 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,www,1732018406961.e85bee36dde7095f8d08f860a32613d7. 2024-11-19T12:13:27,321 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732018406961.e85bee36dde7095f8d08f860a32613d7. 2024-11-19T12:13:27,321 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1732018406961.e85bee36dde7095f8d08f860a32613d7. after waiting 0 ms 2024-11-19T12:13:27,321 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1732018406961.e85bee36dde7095f8d08f860a32613d7. 2024-11-19T12:13:27,321 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,www,1732018406961.e85bee36dde7095f8d08f860a32613d7. 2024-11-19T12:13:27,321 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for e85bee36dde7095f8d08f860a32613d7: Waiting for close lock at 1732018407321Disabling compacts and flushes for region at 1732018407321Disabling writes for close at 1732018407321Writing region close event to WAL at 1732018407321Closed at 1732018407321 2024-11-19T12:13:27,322 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => af2a9e847c9e4d4812af432c436c7ac0, NAME => 'test,yyy,1732018406961.af2a9e847c9e4d4812af432c436c7ac0.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741861_1037 (size=45) 2024-11-19T12:13:27,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741861_1037 (size=45) 2024-11-19T12:13:27,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741861_1037 (size=45) 2024-11-19T12:13:27,326 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,326 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 11d894b29c3ee7cc1e5281c570908571, disabling compactions & flushes 2024-11-19T12:13:27,326 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571. 2024-11-19T12:13:27,326 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571. 2024-11-19T12:13:27,326 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571. after waiting 0 ms 2024-11-19T12:13:27,326 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571. 2024-11-19T12:13:27,326 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571. 2024-11-19T12:13:27,327 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 11d894b29c3ee7cc1e5281c570908571: Waiting for close lock at 1732018407326Disabling compacts and flushes for region at 1732018407326Disabling writes for close at 1732018407326Writing region close event to WAL at 1732018407326Closed at 1732018407326 2024-11-19T12:13:27,327 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 4fea5432239dac4bfb1b607057a95e34, NAME => 'test,zzz,1732018406961.4fea5432239dac4bfb1b607057a95e34.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8 2024-11-19T12:13:27,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741862_1038 (size=45) 2024-11-19T12:13:27,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741862_1038 (size=45) 2024-11-19T12:13:27,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741862_1038 (size=45) 2024-11-19T12:13:27,334 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,yyy,1732018406961.af2a9e847c9e4d4812af432c436c7ac0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,334 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing af2a9e847c9e4d4812af432c436c7ac0, disabling compactions & flushes 2024-11-19T12:13:27,334 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,yyy,1732018406961.af2a9e847c9e4d4812af432c436c7ac0. 2024-11-19T12:13:27,334 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732018406961.af2a9e847c9e4d4812af432c436c7ac0. 2024-11-19T12:13:27,334 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732018406961.af2a9e847c9e4d4812af432c436c7ac0. after waiting 0 ms 2024-11-19T12:13:27,334 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732018406961.af2a9e847c9e4d4812af432c436c7ac0. 2024-11-19T12:13:27,334 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,yyy,1732018406961.af2a9e847c9e4d4812af432c436c7ac0. 2024-11-19T12:13:27,334 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for af2a9e847c9e4d4812af432c436c7ac0: Waiting for close lock at 1732018407334Disabling compacts and flushes for region at 1732018407334Disabling writes for close at 1732018407334Writing region close event to WAL at 1732018407334Closed at 1732018407334 2024-11-19T12:13:27,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741863_1039 (size=42) 2024-11-19T12:13:27,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741863_1039 (size=42) 2024-11-19T12:13:27,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741863_1039 (size=42) 2024-11-19T12:13:27,339 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,zzz,1732018406961.4fea5432239dac4bfb1b607057a95e34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,339 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 4fea5432239dac4bfb1b607057a95e34, disabling compactions & flushes 2024-11-19T12:13:27,339 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,zzz,1732018406961.4fea5432239dac4bfb1b607057a95e34. 2024-11-19T12:13:27,339 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732018406961.4fea5432239dac4bfb1b607057a95e34. 2024-11-19T12:13:27,339 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732018406961.4fea5432239dac4bfb1b607057a95e34. after waiting 0 ms 2024-11-19T12:13:27,339 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732018406961.4fea5432239dac4bfb1b607057a95e34. 2024-11-19T12:13:27,339 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,zzz,1732018406961.4fea5432239dac4bfb1b607057a95e34. 2024-11-19T12:13:27,339 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 4fea5432239dac4bfb1b607057a95e34: Waiting for close lock at 1732018407339Disabling compacts and flushes for region at 1732018407339Disabling writes for close at 1732018407339Writing region close event to WAL at 1732018407339Closed at 1732018407339 2024-11-19T12:13:27,553 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,iii,1732018406961.ab050043367e9fb30539590d5f965d47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,553 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing ab050043367e9fb30539590d5f965d47, disabling compactions & flushes 2024-11-19T12:13:27,553 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,iii,1732018406961.ab050043367e9fb30539590d5f965d47. 2024-11-19T12:13:27,553 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732018406961.ab050043367e9fb30539590d5f965d47. 2024-11-19T12:13:27,553 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732018406961.ab050043367e9fb30539590d5f965d47. after waiting 0 ms 2024-11-19T12:13:27,553 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732018406961.ab050043367e9fb30539590d5f965d47. 2024-11-19T12:13:27,554 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,iii,1732018406961.ab050043367e9fb30539590d5f965d47. 2024-11-19T12:13:27,554 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for ab050043367e9fb30539590d5f965d47: Waiting for close lock at 1732018407553Disabling compacts and flushes for region at 1732018407553Disabling writes for close at 1732018407553Writing region close event to WAL at 1732018407553Closed at 1732018407553 2024-11-19T12:13:27,557 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T12:13:27,563 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1732018406961.5b9b9ba699dcd89ef4b3398f2beb2455.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,563 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1732018406961.b859464d4ffda6f2df23654fb949364f.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,564 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1732018406961.e257775d8750f12b70161bb0b6387163.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,564 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1732018406961.8fbade10b0e1d2be0489df2d6c3fcf9d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,564 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,564 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1732018406961.a78d1de57ec917279b938f869bdbe10d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,564 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1732018406961.125c93438c95e8e5f1b8be66c59d7cf9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,564 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1732018406961.7382ad40044282a358da8130ce023367.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,565 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1732018406961.19add541d34392b48fdd488f1c19c182.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,565 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1732018406961.a329354a61adc5325fa166bf564b84eb.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,565 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,565 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1732018406961.adc3abaf30d887a0d57a7730313a62b4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,565 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1732018406961.ef4298fd6621c8e7facf42422bd80be4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,565 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,565 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1732018406961.963c42e1179165051d86c1560642c124.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,566 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1732018406961.ca6b3c7ec060f344fff683e91e796d33.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,566 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,566 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1732018406961.797a72fe833580fc974f887653d7763a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,566 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1732018406961.2ec97ffd6b2cc998be397eec7fc0a0bc.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,566 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,566 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1732018406961.2b6d3f4b598ab015d9c62b67f0bac50f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,566 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1732018406961.e85bee36dde7095f8d08f860a32613d7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,566 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,567 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1732018406961.af2a9e847c9e4d4812af432c436c7ac0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,567 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1732018406961.4fea5432239dac4bfb1b607057a95e34.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,567 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1732018406961.ab050043367e9fb30539590d5f965d47.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732018407557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732018407557"}]},"ts":"1732018407557"} 2024-11-19T12:13:27,608 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-19T12:13:27,610 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T12:13:27,612 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732018407610"}]},"ts":"1732018407610"} 2024-11-19T12:13:27,617 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-19T12:13:27,618 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {351a98347401=0} racks are {/default-rack=0} 2024-11-19T12:13:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T12:13:27,637 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T12:13:27,637 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T12:13:27,637 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-19T12:13:27,637 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T12:13:27,637 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T12:13:27,637 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-19T12:13:27,637 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T12:13:27,637 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T12:13:27,637 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-19T12:13:27,637 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T12:13:27,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e257775d8750f12b70161bb0b6387163, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5b9b9ba699dcd89ef4b3398f2beb2455, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8fbade10b0e1d2be0489df2d6c3fcf9d, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a78d1de57ec917279b938f869bdbe10d, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=125c93438c95e8e5f1b8be66c59d7cf9, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=19add541d34392b48fdd488f1c19c182, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a329354a61adc5325fa166bf564b84eb, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=adc3abaf30d887a0d57a7730313a62b4, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ef4298fd6621c8e7facf42422bd80be4, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=963c42e1179165051d86c1560642c124, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ca6b3c7ec060f344fff683e91e796d33, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=797a72fe833580fc974f887653d7763a, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2ec97ffd6b2cc998be397eec7fc0a0bc, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2b6d3f4b598ab015d9c62b67f0bac50f, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e85bee36dde7095f8d08f860a32613d7, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=af2a9e847c9e4d4812af432c436c7ac0, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4fea5432239dac4bfb1b607057a95e34, ASSIGN}] 2024-11-19T12:13:27,652 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4fea5432239dac4bfb1b607057a95e34, ASSIGN 2024-11-19T12:13:27,654 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4fea5432239dac4bfb1b607057a95e34, ASSIGN; state=OFFLINE, location=351a98347401,45367,1732018404555; forceNewPlan=false, retain=false 2024-11-19T12:13:27,655 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=af2a9e847c9e4d4812af432c436c7ac0, ASSIGN 2024-11-19T12:13:27,657 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=af2a9e847c9e4d4812af432c436c7ac0, ASSIGN; state=OFFLINE, location=351a98347401,36275,1732018404438; forceNewPlan=false, retain=false 2024-11-19T12:13:27,659 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:27,660 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN; state=OFFLINE, location=351a98347401,35543,1732018404609; forceNewPlan=false, retain=false 2024-11-19T12:13:27,661 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e85bee36dde7095f8d08f860a32613d7, ASSIGN 2024-11-19T12:13:27,663 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e85bee36dde7095f8d08f860a32613d7, ASSIGN; state=OFFLINE, location=351a98347401,45367,1732018404555; forceNewPlan=false, retain=false 2024-11-19T12:13:27,664 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2b6d3f4b598ab015d9c62b67f0bac50f, ASSIGN 2024-11-19T12:13:27,666 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2b6d3f4b598ab015d9c62b67f0bac50f, ASSIGN; state=OFFLINE, location=351a98347401,36275,1732018404438; forceNewPlan=false, retain=false 2024-11-19T12:13:27,667 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:27,668 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN; state=OFFLINE, location=351a98347401,35543,1732018404609; forceNewPlan=false, retain=false 2024-11-19T12:13:27,670 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2ec97ffd6b2cc998be397eec7fc0a0bc, ASSIGN 2024-11-19T12:13:27,671 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2ec97ffd6b2cc998be397eec7fc0a0bc, ASSIGN; state=OFFLINE, location=351a98347401,45367,1732018404555; forceNewPlan=false, retain=false 2024-11-19T12:13:27,672 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=797a72fe833580fc974f887653d7763a, ASSIGN 2024-11-19T12:13:27,673 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=797a72fe833580fc974f887653d7763a, ASSIGN; state=OFFLINE, location=351a98347401,36275,1732018404438; forceNewPlan=false, retain=false 2024-11-19T12:13:27,674 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:27,676 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN; state=OFFLINE, location=351a98347401,35543,1732018404609; forceNewPlan=false, retain=false 2024-11-19T12:13:27,677 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ca6b3c7ec060f344fff683e91e796d33, ASSIGN 2024-11-19T12:13:27,679 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ca6b3c7ec060f344fff683e91e796d33, ASSIGN; state=OFFLINE, location=351a98347401,45367,1732018404555; forceNewPlan=false, retain=false 2024-11-19T12:13:27,680 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=963c42e1179165051d86c1560642c124, ASSIGN 2024-11-19T12:13:27,681 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=963c42e1179165051d86c1560642c124, ASSIGN; state=OFFLINE, location=351a98347401,36275,1732018404438; forceNewPlan=false, retain=false 2024-11-19T12:13:27,682 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:27,684 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN; state=OFFLINE, location=351a98347401,35543,1732018404609; forceNewPlan=false, retain=false 2024-11-19T12:13:27,685 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ef4298fd6621c8e7facf42422bd80be4, ASSIGN 2024-11-19T12:13:27,686 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ef4298fd6621c8e7facf42422bd80be4, ASSIGN; state=OFFLINE, location=351a98347401,45367,1732018404555; forceNewPlan=false, retain=false 2024-11-19T12:13:27,688 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=adc3abaf30d887a0d57a7730313a62b4, ASSIGN 2024-11-19T12:13:27,689 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=adc3abaf30d887a0d57a7730313a62b4, ASSIGN; state=OFFLINE, location=351a98347401,36275,1732018404438; forceNewPlan=false, retain=false 2024-11-19T12:13:27,691 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:27,692 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN; state=OFFLINE, location=351a98347401,35543,1732018404609; forceNewPlan=false, retain=false 2024-11-19T12:13:27,693 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a329354a61adc5325fa166bf564b84eb, ASSIGN 2024-11-19T12:13:27,694 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a329354a61adc5325fa166bf564b84eb, ASSIGN; state=OFFLINE, location=351a98347401,45367,1732018404555; forceNewPlan=false, retain=false 2024-11-19T12:13:27,696 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=19add541d34392b48fdd488f1c19c182, ASSIGN 2024-11-19T12:13:27,697 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=19add541d34392b48fdd488f1c19c182, ASSIGN; state=OFFLINE, location=351a98347401,36275,1732018404438; forceNewPlan=false, retain=false 2024-11-19T12:13:27,698 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:27,699 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN; state=OFFLINE, location=351a98347401,35543,1732018404609; forceNewPlan=false, retain=false 2024-11-19T12:13:27,701 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, ASSIGN 2024-11-19T12:13:27,702 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, ASSIGN; state=OFFLINE, location=351a98347401,35543,1732018404609; forceNewPlan=false, retain=false 2024-11-19T12:13:27,704 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=125c93438c95e8e5f1b8be66c59d7cf9, ASSIGN 2024-11-19T12:13:27,705 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=125c93438c95e8e5f1b8be66c59d7cf9, ASSIGN; state=OFFLINE, location=351a98347401,45367,1732018404555; forceNewPlan=false, retain=false 2024-11-19T12:13:27,706 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a78d1de57ec917279b938f869bdbe10d, ASSIGN 2024-11-19T12:13:27,708 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a78d1de57ec917279b938f869bdbe10d, ASSIGN; state=OFFLINE, location=351a98347401,36275,1732018404438; forceNewPlan=false, retain=false 2024-11-19T12:13:27,709 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN 2024-11-19T12:13:27,710 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN; state=OFFLINE, location=351a98347401,35543,1732018404609; forceNewPlan=false, retain=false 2024-11-19T12:13:27,711 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8fbade10b0e1d2be0489df2d6c3fcf9d, ASSIGN 2024-11-19T12:13:27,713 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8fbade10b0e1d2be0489df2d6c3fcf9d, ASSIGN; state=OFFLINE, location=351a98347401,45367,1732018404555; forceNewPlan=false, retain=false 2024-11-19T12:13:27,714 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5b9b9ba699dcd89ef4b3398f2beb2455, ASSIGN 2024-11-19T12:13:27,715 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5b9b9ba699dcd89ef4b3398f2beb2455, ASSIGN; state=OFFLINE, location=351a98347401,45367,1732018404555; forceNewPlan=false, retain=false 2024-11-19T12:13:27,717 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e257775d8750f12b70161bb0b6387163, ASSIGN 2024-11-19T12:13:27,718 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e257775d8750f12b70161bb0b6387163, ASSIGN; state=OFFLINE, location=351a98347401,36275,1732018404438; forceNewPlan=false, retain=false 2024-11-19T12:13:27,719 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:27,721 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN; state=OFFLINE, location=351a98347401,35543,1732018404609; forceNewPlan=false, retain=false 2024-11-19T12:13:27,811 INFO [351a98347401:38839 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-19T12:13:27,812 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=4fea5432239dac4bfb1b607057a95e34, regionState=OPENING, regionLocation=351a98347401,45367,1732018404555 2024-11-19T12:13:27,813 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=e85bee36dde7095f8d08f860a32613d7, regionState=OPENING, regionLocation=351a98347401,45367,1732018404555 2024-11-19T12:13:27,814 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPENING, regionLocation=351a98347401,45367,1732018404555 2024-11-19T12:13:27,815 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=ca6b3c7ec060f344fff683e91e796d33, regionState=OPENING, regionLocation=351a98347401,45367,1732018404555 2024-11-19T12:13:27,815 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=ef4298fd6621c8e7facf42422bd80be4, regionState=OPENING, regionLocation=351a98347401,45367,1732018404555 2024-11-19T12:13:27,816 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=a329354a61adc5325fa166bf564b84eb, regionState=OPENING, regionLocation=351a98347401,45367,1732018404555 2024-11-19T12:13:27,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4fea5432239dac4bfb1b607057a95e34, ASSIGN because future has completed 2024-11-19T12:13:27,816 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPENING, regionLocation=351a98347401,45367,1732018404555 2024-11-19T12:13:27,817 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPENING, regionLocation=351a98347401,45367,1732018404555 2024-11-19T12:13:27,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4fea5432239dac4bfb1b607057a95e34, server=351a98347401,45367,1732018404555}] 2024-11-19T12:13:27,819 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e85bee36dde7095f8d08f860a32613d7, ASSIGN because future has completed 2024-11-19T12:13:27,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2ec97ffd6b2cc998be397eec7fc0a0bc, ASSIGN because future has completed 2024-11-19T12:13:27,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ec97ffd6b2cc998be397eec7fc0a0bc, server=351a98347401,45367,1732018404555}] 2024-11-19T12:13:27,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ca6b3c7ec060f344fff683e91e796d33, ASSIGN because future has completed 2024-11-19T12:13:27,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ef4298fd6621c8e7facf42422bd80be4, ASSIGN because future has completed 2024-11-19T12:13:27,832 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a329354a61adc5325fa166bf564b84eb, ASSIGN because future has completed 2024-11-19T12:13:27,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure a329354a61adc5325fa166bf564b84eb, server=351a98347401,45367,1732018404555}] 2024-11-19T12:13:27,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=125c93438c95e8e5f1b8be66c59d7cf9, ASSIGN because future has completed 2024-11-19T12:13:27,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8fbade10b0e1d2be0489df2d6c3fcf9d, ASSIGN because future has completed 2024-11-19T12:13:27,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fbade10b0e1d2be0489df2d6c3fcf9d, server=351a98347401,45367,1732018404555}] 2024-11-19T12:13:27,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 125c93438c95e8e5f1b8be66c59d7cf9, server=351a98347401,45367,1732018404555}] 2024-11-19T12:13:27,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef4298fd6621c8e7facf42422bd80be4, server=351a98347401,45367,1732018404555}] 2024-11-19T12:13:27,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca6b3c7ec060f344fff683e91e796d33, server=351a98347401,45367,1732018404555}] 2024-11-19T12:13:27,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure e85bee36dde7095f8d08f860a32613d7, server=351a98347401,45367,1732018404555}] 2024-11-19T12:13:27,869 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPENING, regionLocation=351a98347401,45367,1732018404555 2024-11-19T12:13:27,870 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPENING, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:27,871 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPENING, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:27,871 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=797a72fe833580fc974f887653d7763a, regionState=OPENING, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:27,872 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=963c42e1179165051d86c1560642c124, regionState=OPENING, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:27,872 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5b9b9ba699dcd89ef4b3398f2beb2455, ASSIGN because future has completed 2024-11-19T12:13:27,873 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=adc3abaf30d887a0d57a7730313a62b4, regionState=OPENING, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:27,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b9b9ba699dcd89ef4b3398f2beb2455, server=351a98347401,45367,1732018404555}] 2024-11-19T12:13:27,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=af2a9e847c9e4d4812af432c436c7ac0, ASSIGN because future has completed 2024-11-19T12:13:27,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2b6d3f4b598ab015d9c62b67f0bac50f, ASSIGN because future has completed 2024-11-19T12:13:27,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=797a72fe833580fc974f887653d7763a, ASSIGN because future has completed 2024-11-19T12:13:27,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 797a72fe833580fc974f887653d7763a, server=351a98347401,36275,1732018404438}] 2024-11-19T12:13:27,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=963c42e1179165051d86c1560642c124, ASSIGN because future has completed 2024-11-19T12:13:27,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=adc3abaf30d887a0d57a7730313a62b4, ASSIGN because future has completed 2024-11-19T12:13:27,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure adc3abaf30d887a0d57a7730313a62b4, server=351a98347401,36275,1732018404438}] 2024-11-19T12:13:27,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 963c42e1179165051d86c1560642c124, server=351a98347401,36275,1732018404438}] 2024-11-19T12:13:27,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6d3f4b598ab015d9c62b67f0bac50f, server=351a98347401,36275,1732018404438}] 2024-11-19T12:13:27,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure af2a9e847c9e4d4812af432c436c7ac0, server=351a98347401,36275,1732018404438}] 2024-11-19T12:13:27,901 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=19add541d34392b48fdd488f1c19c182, regionState=OPENING, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:27,902 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=a78d1de57ec917279b938f869bdbe10d, regionState=OPENING, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:27,903 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=e257775d8750f12b70161bb0b6387163, regionState=OPENING, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:27,904 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, regionLocation=351a98347401,35543,1732018404609 2024-11-19T12:13:27,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=19add541d34392b48fdd488f1c19c182, ASSIGN because future has completed 2024-11-19T12:13:27,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 19add541d34392b48fdd488f1c19c182, server=351a98347401,36275,1732018404438}] 2024-11-19T12:13:27,906 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a78d1de57ec917279b938f869bdbe10d, ASSIGN because future has completed 2024-11-19T12:13:27,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e257775d8750f12b70161bb0b6387163, ASSIGN because future has completed 2024-11-19T12:13:27,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN because future has completed 2024-11-19T12:13:27,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11d894b29c3ee7cc1e5281c570908571, server=351a98347401,35543,1732018404609}] 2024-11-19T12:13:27,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure e257775d8750f12b70161bb0b6387163, server=351a98347401,36275,1732018404438}] 2024-11-19T12:13:27,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure a78d1de57ec917279b938f869bdbe10d, server=351a98347401,36275,1732018404438}] 2024-11-19T12:13:27,923 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, regionLocation=351a98347401,35543,1732018404609 2024-11-19T12:13:27,924 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, regionLocation=351a98347401,35543,1732018404609 2024-11-19T12:13:27,924 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, regionLocation=351a98347401,35543,1732018404609 2024-11-19T12:13:27,925 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, regionLocation=351a98347401,35543,1732018404609 2024-11-19T12:13:27,925 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=ab050043367e9fb30539590d5f965d47, regionState=OPENING, regionLocation=351a98347401,35543,1732018404609 2024-11-19T12:13:27,926 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=7382ad40044282a358da8130ce023367, regionState=OPENING, regionLocation=351a98347401,35543,1732018404609 2024-11-19T12:13:27,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN because future has completed 2024-11-19T12:13:27,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, server=351a98347401,35543,1732018404609}] 2024-11-19T12:13:27,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN because future has completed 2024-11-19T12:13:27,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN because future has completed 2024-11-19T12:13:27,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN because future has completed 2024-11-19T12:13:27,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN because future has completed 2024-11-19T12:13:27,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, ASSIGN because future has completed 2024-11-19T12:13:27,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7382ad40044282a358da8130ce023367, server=351a98347401,35543,1732018404609}] 2024-11-19T12:13:27,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab050043367e9fb30539590d5f965d47, server=351a98347401,35543,1732018404609}] 2024-11-19T12:13:27,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bc8ba883b6bd714e90daec2cb61209e, server=351a98347401,35543,1732018404609}] 2024-11-19T12:13:27,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8bfccd6e991fb7a90ce5bcadc0a3ef5, server=351a98347401,35543,1732018404609}] 2024-11-19T12:13:27,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24b360a2b049dbc9c784cdd687321115, server=351a98347401,35543,1732018404609}] 2024-11-19T12:13:27,957 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, regionLocation=351a98347401,35543,1732018404609 2024-11-19T12:13:27,957 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, regionLocation=351a98347401,35543,1732018404609 2024-11-19T12:13:27,959 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN because future has completed 2024-11-19T12:13:27,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9224ed53fa7acfead92d4143c8626ff, server=351a98347401,35543,1732018404609}] 2024-11-19T12:13:27,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN because future has completed 2024-11-19T12:13:27,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,35543,1732018404609}] 2024-11-19T12:13:27,973 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T12:13:27,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40011, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T12:13:27,980 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,ggg,1732018406961.125c93438c95e8e5f1b8be66c59d7cf9. 2024-11-19T12:13:27,980 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 125c93438c95e8e5f1b8be66c59d7cf9, NAME => 'test,ggg,1732018406961.125c93438c95e8e5f1b8be66c59d7cf9.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-19T12:13:27,981 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 125c93438c95e8e5f1b8be66c59d7cf9 2024-11-19T12:13:27,981 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,ggg,1732018406961.125c93438c95e8e5f1b8be66c59d7cf9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:27,981 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 125c93438c95e8e5f1b8be66c59d7cf9 2024-11-19T12:13:27,981 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 125c93438c95e8e5f1b8be66c59d7cf9 2024-11-19T12:13:27,983 INFO [StoreOpener-125c93438c95e8e5f1b8be66c59d7cf9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 125c93438c95e8e5f1b8be66c59d7cf9 2024-11-19T12:13:27,985 INFO [StoreOpener-125c93438c95e8e5f1b8be66c59d7cf9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 125c93438c95e8e5f1b8be66c59d7cf9 columnFamilyName family 2024-11-19T12:13:27,985 DEBUG [StoreOpener-125c93438c95e8e5f1b8be66c59d7cf9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:27,986 INFO [StoreOpener-125c93438c95e8e5f1b8be66c59d7cf9-1 {}] regionserver.HStore(327): Store=125c93438c95e8e5f1b8be66c59d7cf9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:27,987 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 125c93438c95e8e5f1b8be66c59d7cf9 2024-11-19T12:13:27,988 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/125c93438c95e8e5f1b8be66c59d7cf9 2024-11-19T12:13:27,989 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/125c93438c95e8e5f1b8be66c59d7cf9 2024-11-19T12:13:27,989 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 125c93438c95e8e5f1b8be66c59d7cf9 2024-11-19T12:13:27,989 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 125c93438c95e8e5f1b8be66c59d7cf9 2024-11-19T12:13:27,992 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 125c93438c95e8e5f1b8be66c59d7cf9 2024-11-19T12:13:27,996 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/125c93438c95e8e5f1b8be66c59d7cf9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:27,997 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 125c93438c95e8e5f1b8be66c59d7cf9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75315706, jitterRate=0.12229147553443909}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:27,997 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 125c93438c95e8e5f1b8be66c59d7cf9 2024-11-19T12:13:27,999 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 125c93438c95e8e5f1b8be66c59d7cf9: Running coprocessor pre-open hook at 1732018407981Writing region info on filesystem at 1732018407981Initializing all the Stores at 1732018407983 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018407983Cleaning up temporary data from old regions at 1732018407989 (+6 ms)Running coprocessor post-open hooks at 1732018407997 (+8 ms)Region opened successfully at 1732018407999 (+2 ms) 2024-11-19T12:13:28,001 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732018406961.125c93438c95e8e5f1b8be66c59d7cf9., pid=35, masterSystemTime=1732018407972 2024-11-19T12:13:28,004 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732018406961.125c93438c95e8e5f1b8be66c59d7cf9. 2024-11-19T12:13:28,004 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,ggg,1732018406961.125c93438c95e8e5f1b8be66c59d7cf9. 2024-11-19T12:13:28,005 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,www,1732018406961.e85bee36dde7095f8d08f860a32613d7. 2024-11-19T12:13:28,005 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => e85bee36dde7095f8d08f860a32613d7, NAME => 'test,www,1732018406961.e85bee36dde7095f8d08f860a32613d7.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-19T12:13:28,005 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,45367,1732018404555 2024-11-19T12:13:28,005 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e85bee36dde7095f8d08f860a32613d7 2024-11-19T12:13:28,006 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,www,1732018406961.e85bee36dde7095f8d08f860a32613d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,006 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for e85bee36dde7095f8d08f860a32613d7 2024-11-19T12:13:28,006 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for e85bee36dde7095f8d08f860a32613d7 2024-11-19T12:13:28,009 INFO [StoreOpener-e85bee36dde7095f8d08f860a32613d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e85bee36dde7095f8d08f860a32613d7 2024-11-19T12:13:28,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 125c93438c95e8e5f1b8be66c59d7cf9, server=351a98347401,45367,1732018404555 because future has completed 2024-11-19T12:13:28,011 INFO [StoreOpener-e85bee36dde7095f8d08f860a32613d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e85bee36dde7095f8d08f860a32613d7 columnFamilyName family 2024-11-19T12:13:28,011 DEBUG [StoreOpener-e85bee36dde7095f8d08f860a32613d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,012 INFO [StoreOpener-e85bee36dde7095f8d08f860a32613d7-1 {}] regionserver.HStore(327): Store=e85bee36dde7095f8d08f860a32613d7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,012 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for e85bee36dde7095f8d08f860a32613d7 2024-11-19T12:13:28,013 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/e85bee36dde7095f8d08f860a32613d7 2024-11-19T12:13:28,014 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/e85bee36dde7095f8d08f860a32613d7 2024-11-19T12:13:28,015 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=11 2024-11-19T12:13:28,015 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 125c93438c95e8e5f1b8be66c59d7cf9, server=351a98347401,45367,1732018404555 in 160 msec 2024-11-19T12:13:28,015 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for e85bee36dde7095f8d08f860a32613d7 2024-11-19T12:13:28,015 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for e85bee36dde7095f8d08f860a32613d7 2024-11-19T12:13:28,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=125c93438c95e8e5f1b8be66c59d7cf9, ASSIGN in 375 msec 2024-11-19T12:13:28,018 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for e85bee36dde7095f8d08f860a32613d7 2024-11-19T12:13:28,021 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/e85bee36dde7095f8d08f860a32613d7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,022 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened e85bee36dde7095f8d08f860a32613d7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71885335, jitterRate=0.07117496430873871}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,022 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e85bee36dde7095f8d08f860a32613d7 2024-11-19T12:13:28,022 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for e85bee36dde7095f8d08f860a32613d7: Running coprocessor pre-open hook at 1732018408006Writing region info on filesystem at 1732018408006Initializing all the Stores at 1732018408008 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408008Cleaning up temporary data from old regions at 1732018408015 (+7 ms)Running coprocessor post-open hooks at 1732018408022 (+7 ms)Region opened successfully at 1732018408022 2024-11-19T12:13:28,023 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732018406961.e85bee36dde7095f8d08f860a32613d7., pid=38, masterSystemTime=1732018407972 2024-11-19T12:13:28,026 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732018406961.e85bee36dde7095f8d08f860a32613d7. 2024-11-19T12:13:28,026 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,www,1732018406961.e85bee36dde7095f8d08f860a32613d7. 2024-11-19T12:13:28,026 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,ddd,1732018406961.8fbade10b0e1d2be0489df2d6c3fcf9d. 2024-11-19T12:13:28,027 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 8fbade10b0e1d2be0489df2d6c3fcf9d, NAME => 'test,ddd,1732018406961.8fbade10b0e1d2be0489df2d6c3fcf9d.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-19T12:13:28,027 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,45367,1732018404555 2024-11-19T12:13:28,027 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8fbade10b0e1d2be0489df2d6c3fcf9d 2024-11-19T12:13:28,027 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,ddd,1732018406961.8fbade10b0e1d2be0489df2d6c3fcf9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,027 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 8fbade10b0e1d2be0489df2d6c3fcf9d 2024-11-19T12:13:28,027 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 8fbade10b0e1d2be0489df2d6c3fcf9d 2024-11-19T12:13:28,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure e85bee36dde7095f8d08f860a32613d7, server=351a98347401,45367,1732018404555 because future has completed 2024-11-19T12:13:28,030 INFO [StoreOpener-8fbade10b0e1d2be0489df2d6c3fcf9d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8fbade10b0e1d2be0489df2d6c3fcf9d 2024-11-19T12:13:28,033 INFO [StoreOpener-8fbade10b0e1d2be0489df2d6c3fcf9d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8fbade10b0e1d2be0489df2d6c3fcf9d columnFamilyName family 2024-11-19T12:13:28,033 DEBUG [StoreOpener-8fbade10b0e1d2be0489df2d6c3fcf9d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,033 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T12:13:28,034 INFO [StoreOpener-8fbade10b0e1d2be0489df2d6c3fcf9d-1 {}] regionserver.HStore(327): Store=8fbade10b0e1d2be0489df2d6c3fcf9d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,034 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 8fbade10b0e1d2be0489df2d6c3fcf9d 2024-11-19T12:13:28,036 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50105, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T12:13:28,036 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/8fbade10b0e1d2be0489df2d6c3fcf9d 2024-11-19T12:13:28,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=27 2024-11-19T12:13:28,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure e85bee36dde7095f8d08f860a32613d7, server=351a98347401,45367,1732018404555 in 166 msec 2024-11-19T12:13:28,037 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/8fbade10b0e1d2be0489df2d6c3fcf9d 2024-11-19T12:13:28,037 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 8fbade10b0e1d2be0489df2d6c3fcf9d 2024-11-19T12:13:28,037 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 8fbade10b0e1d2be0489df2d6c3fcf9d 2024-11-19T12:13:28,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e85bee36dde7095f8d08f860a32613d7, ASSIGN in 397 msec 2024-11-19T12:13:28,040 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 8fbade10b0e1d2be0489df2d6c3fcf9d 2024-11-19T12:13:28,041 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,mmm,1732018406961.adc3abaf30d887a0d57a7730313a62b4. 2024-11-19T12:13:28,042 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => adc3abaf30d887a0d57a7730313a62b4, NAME => 'test,mmm,1732018406961.adc3abaf30d887a0d57a7730313a62b4.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-19T12:13:28,042 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test adc3abaf30d887a0d57a7730313a62b4 2024-11-19T12:13:28,042 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,mmm,1732018406961.adc3abaf30d887a0d57a7730313a62b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,043 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for adc3abaf30d887a0d57a7730313a62b4 2024-11-19T12:13:28,043 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for adc3abaf30d887a0d57a7730313a62b4 2024-11-19T12:13:28,044 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/8fbade10b0e1d2be0489df2d6c3fcf9d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,045 INFO [StoreOpener-adc3abaf30d887a0d57a7730313a62b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region adc3abaf30d887a0d57a7730313a62b4 2024-11-19T12:13:28,045 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 8fbade10b0e1d2be0489df2d6c3fcf9d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71543046, jitterRate=0.06607446074485779}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,045 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8fbade10b0e1d2be0489df2d6c3fcf9d 2024-11-19T12:13:28,046 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 8fbade10b0e1d2be0489df2d6c3fcf9d: Running coprocessor pre-open hook at 1732018408028Writing region info on filesystem at 1732018408028Initializing all the Stores at 1732018408030 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408030Cleaning up temporary data from old regions at 1732018408037 (+7 ms)Running coprocessor post-open hooks at 1732018408045 (+8 ms)Region opened successfully at 1732018408045 2024-11-19T12:13:28,047 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732018406961.8fbade10b0e1d2be0489df2d6c3fcf9d., pid=34, masterSystemTime=1732018407972 2024-11-19T12:13:28,048 INFO [StoreOpener-adc3abaf30d887a0d57a7730313a62b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adc3abaf30d887a0d57a7730313a62b4 columnFamilyName family 2024-11-19T12:13:28,048 DEBUG [StoreOpener-adc3abaf30d887a0d57a7730313a62b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,049 INFO [StoreOpener-adc3abaf30d887a0d57a7730313a62b4-1 {}] regionserver.HStore(327): Store=adc3abaf30d887a0d57a7730313a62b4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,049 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for adc3abaf30d887a0d57a7730313a62b4 2024-11-19T12:13:28,050 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732018406961.8fbade10b0e1d2be0489df2d6c3fcf9d. 2024-11-19T12:13:28,050 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,ddd,1732018406961.8fbade10b0e1d2be0489df2d6c3fcf9d. 2024-11-19T12:13:28,050 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,qqq,1732018406961.ca6b3c7ec060f344fff683e91e796d33. 2024-11-19T12:13:28,050 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => ca6b3c7ec060f344fff683e91e796d33, NAME => 'test,qqq,1732018406961.ca6b3c7ec060f344fff683e91e796d33.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-19T12:13:28,050 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,45367,1732018404555 2024-11-19T12:13:28,051 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ca6b3c7ec060f344fff683e91e796d33 2024-11-19T12:13:28,051 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/adc3abaf30d887a0d57a7730313a62b4 2024-11-19T12:13:28,051 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,qqq,1732018406961.ca6b3c7ec060f344fff683e91e796d33.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,051 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for ca6b3c7ec060f344fff683e91e796d33 2024-11-19T12:13:28,051 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for ca6b3c7ec060f344fff683e91e796d33 2024-11-19T12:13:28,051 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/adc3abaf30d887a0d57a7730313a62b4 2024-11-19T12:13:28,052 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for adc3abaf30d887a0d57a7730313a62b4 2024-11-19T12:13:28,053 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for adc3abaf30d887a0d57a7730313a62b4 2024-11-19T12:13:28,053 INFO [StoreOpener-ca6b3c7ec060f344fff683e91e796d33-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ca6b3c7ec060f344fff683e91e796d33 2024-11-19T12:13:28,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fbade10b0e1d2be0489df2d6c3fcf9d, server=351a98347401,45367,1732018404555 because future has completed 2024-11-19T12:13:28,056 INFO [StoreOpener-ca6b3c7ec060f344fff683e91e796d33-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca6b3c7ec060f344fff683e91e796d33 columnFamilyName family 2024-11-19T12:13:28,055 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for adc3abaf30d887a0d57a7730313a62b4 2024-11-19T12:13:28,056 DEBUG [StoreOpener-ca6b3c7ec060f344fff683e91e796d33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,056 INFO [StoreOpener-ca6b3c7ec060f344fff683e91e796d33-1 {}] regionserver.HStore(327): Store=ca6b3c7ec060f344fff683e91e796d33/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,057 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for ca6b3c7ec060f344fff683e91e796d33 2024-11-19T12:13:28,058 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/ca6b3c7ec060f344fff683e91e796d33 2024-11-19T12:13:28,059 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/ca6b3c7ec060f344fff683e91e796d33 2024-11-19T12:13:28,059 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for ca6b3c7ec060f344fff683e91e796d33 2024-11-19T12:13:28,060 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for ca6b3c7ec060f344fff683e91e796d33 2024-11-19T12:13:28,060 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/adc3abaf30d887a0d57a7730313a62b4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=8 2024-11-19T12:13:28,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 8fbade10b0e1d2be0489df2d6c3fcf9d, server=351a98347401,45367,1732018404555 in 211 msec 2024-11-19T12:13:28,061 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened adc3abaf30d887a0d57a7730313a62b4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71914920, jitterRate=0.07161581516265869}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,061 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for adc3abaf30d887a0d57a7730313a62b4 2024-11-19T12:13:28,061 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for adc3abaf30d887a0d57a7730313a62b4: Running coprocessor pre-open hook at 1732018408043Writing region info on filesystem at 1732018408043Initializing all the Stores at 1732018408044 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408044Cleaning up temporary data from old regions at 1732018408053 (+9 ms)Running coprocessor post-open hooks at 1732018408061 (+8 ms)Region opened successfully at 1732018408061 2024-11-19T12:13:28,063 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732018406961.adc3abaf30d887a0d57a7730313a62b4., pid=41, masterSystemTime=1732018408033 2024-11-19T12:13:28,064 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for ca6b3c7ec060f344fff683e91e796d33 2024-11-19T12:13:28,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8fbade10b0e1d2be0489df2d6c3fcf9d, ASSIGN in 421 msec 2024-11-19T12:13:28,068 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/ca6b3c7ec060f344fff683e91e796d33/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,069 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732018406961.adc3abaf30d887a0d57a7730313a62b4. 2024-11-19T12:13:28,069 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,mmm,1732018406961.adc3abaf30d887a0d57a7730313a62b4. 2024-11-19T12:13:28,069 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,ppp,1732018406961.963c42e1179165051d86c1560642c124. 2024-11-19T12:13:28,069 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened ca6b3c7ec060f344fff683e91e796d33; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59533546, jitterRate=-0.11288103461265564}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,069 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ca6b3c7ec060f344fff683e91e796d33 2024-11-19T12:13:28,069 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 963c42e1179165051d86c1560642c124, NAME => 'test,ppp,1732018406961.963c42e1179165051d86c1560642c124.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-19T12:13:28,070 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for ca6b3c7ec060f344fff683e91e796d33: Running coprocessor pre-open hook at 1732018408051Writing region info on filesystem at 1732018408051Initializing all the Stores at 1732018408053 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408053Cleaning up temporary data from old regions at 1732018408060 (+7 ms)Running coprocessor post-open hooks at 1732018408069 (+9 ms)Region opened successfully at 1732018408069 2024-11-19T12:13:28,070 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:28,070 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 963c42e1179165051d86c1560642c124 2024-11-19T12:13:28,070 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,ppp,1732018406961.963c42e1179165051d86c1560642c124.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,070 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 963c42e1179165051d86c1560642c124 2024-11-19T12:13:28,070 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 963c42e1179165051d86c1560642c124 2024-11-19T12:13:28,072 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732018406961.ca6b3c7ec060f344fff683e91e796d33., pid=37, masterSystemTime=1732018407972 2024-11-19T12:13:28,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure adc3abaf30d887a0d57a7730313a62b4, server=351a98347401,36275,1732018404438 because future has completed 2024-11-19T12:13:28,075 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732018406961.ca6b3c7ec060f344fff683e91e796d33. 2024-11-19T12:13:28,075 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,qqq,1732018406961.ca6b3c7ec060f344fff683e91e796d33. 2024-11-19T12:13:28,075 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,ccc,1732018406961.5b9b9ba699dcd89ef4b3398f2beb2455. 2024-11-19T12:13:28,075 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 5b9b9ba699dcd89ef4b3398f2beb2455, NAME => 'test,ccc,1732018406961.5b9b9ba699dcd89ef4b3398f2beb2455.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-19T12:13:28,076 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5b9b9ba699dcd89ef4b3398f2beb2455 2024-11-19T12:13:28,076 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,ccc,1732018406961.5b9b9ba699dcd89ef4b3398f2beb2455.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,076 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 5b9b9ba699dcd89ef4b3398f2beb2455 2024-11-19T12:13:28,076 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 5b9b9ba699dcd89ef4b3398f2beb2455 2024-11-19T12:13:28,076 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5. 2024-11-19T12:13:28,077 INFO [StoreOpener-963c42e1179165051d86c1560642c124-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 963c42e1179165051d86c1560642c124 2024-11-19T12:13:28,077 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => d8bfccd6e991fb7a90ce5bcadc0a3ef5, NAME => 'test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-19T12:13:28,078 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:28,078 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,078 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:28,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=17 2024-11-19T12:13:28,079 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:28,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure adc3abaf30d887a0d57a7730313a62b4, server=351a98347401,36275,1732018404438 in 191 msec 2024-11-19T12:13:28,080 INFO [StoreOpener-963c42e1179165051d86c1560642c124-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 963c42e1179165051d86c1560642c124 columnFamilyName family 2024-11-19T12:13:28,080 DEBUG [StoreOpener-963c42e1179165051d86c1560642c124-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,080 INFO [StoreOpener-5b9b9ba699dcd89ef4b3398f2beb2455-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5b9b9ba699dcd89ef4b3398f2beb2455 2024-11-19T12:13:28,081 INFO [StoreOpener-963c42e1179165051d86c1560642c124-1 {}] regionserver.HStore(327): Store=963c42e1179165051d86c1560642c124/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,081 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 963c42e1179165051d86c1560642c124 2024-11-19T12:13:28,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=adc3abaf30d887a0d57a7730313a62b4, ASSIGN in 439 msec 2024-11-19T12:13:28,082 INFO [StoreOpener-d8bfccd6e991fb7a90ce5bcadc0a3ef5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:28,084 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/963c42e1179165051d86c1560642c124 2024-11-19T12:13:28,084 INFO [StoreOpener-5b9b9ba699dcd89ef4b3398f2beb2455-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b9b9ba699dcd89ef4b3398f2beb2455 columnFamilyName family 2024-11-19T12:13:28,084 DEBUG [StoreOpener-5b9b9ba699dcd89ef4b3398f2beb2455-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,085 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,45367,1732018404555 2024-11-19T12:13:28,085 INFO [StoreOpener-d8bfccd6e991fb7a90ce5bcadc0a3ef5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d8bfccd6e991fb7a90ce5bcadc0a3ef5 columnFamilyName family 2024-11-19T12:13:28,085 DEBUG [StoreOpener-d8bfccd6e991fb7a90ce5bcadc0a3ef5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,085 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/963c42e1179165051d86c1560642c124 2024-11-19T12:13:28,085 INFO [StoreOpener-5b9b9ba699dcd89ef4b3398f2beb2455-1 {}] regionserver.HStore(327): Store=5b9b9ba699dcd89ef4b3398f2beb2455/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,086 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 5b9b9ba699dcd89ef4b3398f2beb2455 2024-11-19T12:13:28,086 INFO [StoreOpener-d8bfccd6e991fb7a90ce5bcadc0a3ef5-1 {}] regionserver.HStore(327): Store=d8bfccd6e991fb7a90ce5bcadc0a3ef5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,086 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:28,086 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 963c42e1179165051d86c1560642c124 2024-11-19T12:13:28,087 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 963c42e1179165051d86c1560642c124 2024-11-19T12:13:28,087 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:28,088 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/5b9b9ba699dcd89ef4b3398f2beb2455 2024-11-19T12:13:28,088 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:28,089 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:28,089 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:28,089 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/5b9b9ba699dcd89ef4b3398f2beb2455 2024-11-19T12:13:28,090 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca6b3c7ec060f344fff683e91e796d33, server=351a98347401,45367,1732018404555 because future has completed 2024-11-19T12:13:28,091 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 5b9b9ba699dcd89ef4b3398f2beb2455 2024-11-19T12:13:28,091 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 5b9b9ba699dcd89ef4b3398f2beb2455 2024-11-19T12:13:28,092 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 963c42e1179165051d86c1560642c124 2024-11-19T12:13:28,093 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:28,096 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 5b9b9ba699dcd89ef4b3398f2beb2455 2024-11-19T12:13:28,096 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/963c42e1179165051d86c1560642c124/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=21 2024-11-19T12:13:28,097 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 963c42e1179165051d86c1560642c124; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58734783, jitterRate=-0.12478353083133698}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure ca6b3c7ec060f344fff683e91e796d33, server=351a98347401,45367,1732018404555 in 232 msec 2024-11-19T12:13:28,097 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 963c42e1179165051d86c1560642c124 2024-11-19T12:13:28,097 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 963c42e1179165051d86c1560642c124: Running coprocessor pre-open hook at 1732018408070Writing region info on filesystem at 1732018408070Initializing all the Stores at 1732018408072 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408072Cleaning up temporary data from old regions at 1732018408087 (+15 ms)Running coprocessor post-open hooks at 1732018408097 (+10 ms)Region opened successfully at 1732018408097 2024-11-19T12:13:28,099 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732018406961.963c42e1179165051d86c1560642c124., pid=42, masterSystemTime=1732018408033 2024-11-19T12:13:28,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ca6b3c7ec060f344fff683e91e796d33, ASSIGN in 457 msec 2024-11-19T12:13:28,102 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/5b9b9ba699dcd89ef4b3398f2beb2455/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,102 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732018406961.963c42e1179165051d86c1560642c124. 2024-11-19T12:13:28,102 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,ppp,1732018406961.963c42e1179165051d86c1560642c124. 2024-11-19T12:13:28,102 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,yyy,1732018406961.af2a9e847c9e4d4812af432c436c7ac0. 2024-11-19T12:13:28,103 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => af2a9e847c9e4d4812af432c436c7ac0, NAME => 'test,yyy,1732018406961.af2a9e847c9e4d4812af432c436c7ac0.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-19T12:13:28,103 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test af2a9e847c9e4d4812af432c436c7ac0 2024-11-19T12:13:28,103 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,yyy,1732018406961.af2a9e847c9e4d4812af432c436c7ac0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,103 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=963c42e1179165051d86c1560642c124, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:28,103 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for af2a9e847c9e4d4812af432c436c7ac0 2024-11-19T12:13:28,103 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for af2a9e847c9e4d4812af432c436c7ac0 2024-11-19T12:13:28,104 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 5b9b9ba699dcd89ef4b3398f2beb2455; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70254962, jitterRate=0.046880513429641724}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,104 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5b9b9ba699dcd89ef4b3398f2beb2455 2024-11-19T12:13:28,105 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 5b9b9ba699dcd89ef4b3398f2beb2455: Running coprocessor pre-open hook at 1732018408076Writing region info on filesystem at 1732018408076Initializing all the Stores at 1732018408078 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408078Cleaning up temporary data from old regions at 1732018408091 (+13 ms)Running coprocessor post-open hooks at 1732018408104 (+13 ms)Region opened successfully at 1732018408104 2024-11-19T12:13:28,106 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/d8bfccd6e991fb7a90ce5bcadc0a3ef5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,106 INFO [StoreOpener-af2a9e847c9e4d4812af432c436c7ac0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region af2a9e847c9e4d4812af432c436c7ac0 2024-11-19T12:13:28,106 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732018406961.5b9b9ba699dcd89ef4b3398f2beb2455., pid=39, masterSystemTime=1732018407972 2024-11-19T12:13:28,107 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened d8bfccd6e991fb7a90ce5bcadc0a3ef5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65514470, jitterRate=-0.02375832200050354}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,108 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:28,108 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for d8bfccd6e991fb7a90ce5bcadc0a3ef5: Running coprocessor pre-open hook at 1732018408079Writing region info on filesystem at 1732018408079Initializing all the Stores at 1732018408080 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408080Cleaning up temporary data from old regions at 1732018408089 (+9 ms)Running coprocessor post-open hooks at 1732018408108 (+19 ms)Region opened successfully at 1732018408108 2024-11-19T12:13:28,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 963c42e1179165051d86c1560642c124, server=351a98347401,36275,1732018404438 because future has completed 2024-11-19T12:13:28,109 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5., pid=53, masterSystemTime=1732018408062 2024-11-19T12:13:28,110 INFO [StoreOpener-af2a9e847c9e4d4812af432c436c7ac0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af2a9e847c9e4d4812af432c436c7ac0 columnFamilyName family 2024-11-19T12:13:28,110 DEBUG [StoreOpener-af2a9e847c9e4d4812af432c436c7ac0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,110 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732018406961.5b9b9ba699dcd89ef4b3398f2beb2455. 2024-11-19T12:13:28,110 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,ccc,1732018406961.5b9b9ba699dcd89ef4b3398f2beb2455. 2024-11-19T12:13:28,110 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,ttt,1732018406961.2ec97ffd6b2cc998be397eec7fc0a0bc. 2024-11-19T12:13:28,110 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 2ec97ffd6b2cc998be397eec7fc0a0bc, NAME => 'test,ttt,1732018406961.2ec97ffd6b2cc998be397eec7fc0a0bc.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-19T12:13:28,111 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2ec97ffd6b2cc998be397eec7fc0a0bc 2024-11-19T12:13:28,111 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,ttt,1732018406961.2ec97ffd6b2cc998be397eec7fc0a0bc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,111 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 2ec97ffd6b2cc998be397eec7fc0a0bc 2024-11-19T12:13:28,111 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 2ec97ffd6b2cc998be397eec7fc0a0bc 2024-11-19T12:13:28,111 INFO [StoreOpener-af2a9e847c9e4d4812af432c436c7ac0-1 {}] regionserver.HStore(327): Store=af2a9e847c9e4d4812af432c436c7ac0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,111 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for af2a9e847c9e4d4812af432c436c7ac0 2024-11-19T12:13:28,112 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5. 2024-11-19T12:13:28,112 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5. 2024-11-19T12:13:28,113 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,hhh,1732018406961.7382ad40044282a358da8130ce023367. 2024-11-19T12:13:28,113 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/af2a9e847c9e4d4812af432c436c7ac0 2024-11-19T12:13:28,113 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 7382ad40044282a358da8130ce023367, NAME => 'test,hhh,1732018406961.7382ad40044282a358da8130ce023367.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-19T12:13:28,113 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7382ad40044282a358da8130ce023367 2024-11-19T12:13:28,113 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,hhh,1732018406961.7382ad40044282a358da8130ce023367.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,113 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 7382ad40044282a358da8130ce023367 2024-11-19T12:13:28,113 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/af2a9e847c9e4d4812af432c436c7ac0 2024-11-19T12:13:28,113 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 7382ad40044282a358da8130ce023367 2024-11-19T12:13:28,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=20 2024-11-19T12:13:28,114 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for af2a9e847c9e4d4812af432c436c7ac0 2024-11-19T12:13:28,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 963c42e1179165051d86c1560642c124, server=351a98347401,36275,1732018404438 in 221 msec 2024-11-19T12:13:28,114 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for af2a9e847c9e4d4812af432c436c7ac0 2024-11-19T12:13:28,116 INFO [StoreOpener-7382ad40044282a358da8130ce023367-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7382ad40044282a358da8130ce023367 2024-11-19T12:13:28,116 INFO [StoreOpener-2ec97ffd6b2cc998be397eec7fc0a0bc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2ec97ffd6b2cc998be397eec7fc0a0bc 2024-11-19T12:13:28,118 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for af2a9e847c9e4d4812af432c436c7ac0 2024-11-19T12:13:28,119 INFO [StoreOpener-2ec97ffd6b2cc998be397eec7fc0a0bc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ec97ffd6b2cc998be397eec7fc0a0bc columnFamilyName family 2024-11-19T12:13:28,119 DEBUG [StoreOpener-2ec97ffd6b2cc998be397eec7fc0a0bc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,119 INFO [StoreOpener-7382ad40044282a358da8130ce023367-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7382ad40044282a358da8130ce023367 columnFamilyName family 2024-11-19T12:13:28,119 DEBUG [StoreOpener-7382ad40044282a358da8130ce023367-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,120 INFO [StoreOpener-2ec97ffd6b2cc998be397eec7fc0a0bc-1 {}] regionserver.HStore(327): Store=2ec97ffd6b2cc998be397eec7fc0a0bc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,120 INFO [StoreOpener-7382ad40044282a358da8130ce023367-1 {}] regionserver.HStore(327): Store=7382ad40044282a358da8130ce023367/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=963c42e1179165051d86c1560642c124, ASSIGN in 475 msec 2024-11-19T12:13:28,120 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 7382ad40044282a358da8130ce023367 2024-11-19T12:13:28,120 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 2ec97ffd6b2cc998be397eec7fc0a0bc 2024-11-19T12:13:28,121 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/2ec97ffd6b2cc998be397eec7fc0a0bc 2024-11-19T12:13:28,121 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/7382ad40044282a358da8130ce023367 2024-11-19T12:13:28,122 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,35543,1732018404609 2024-11-19T12:13:28,122 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/af2a9e847c9e4d4812af432c436c7ac0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,122 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/7382ad40044282a358da8130ce023367 2024-11-19T12:13:28,122 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/2ec97ffd6b2cc998be397eec7fc0a0bc 2024-11-19T12:13:28,123 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 7382ad40044282a358da8130ce023367 2024-11-19T12:13:28,123 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened af2a9e847c9e4d4812af432c436c7ac0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60181679, jitterRate=-0.10322310030460358}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,123 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 7382ad40044282a358da8130ce023367 2024-11-19T12:13:28,123 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for af2a9e847c9e4d4812af432c436c7ac0 2024-11-19T12:13:28,123 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 2ec97ffd6b2cc998be397eec7fc0a0bc 2024-11-19T12:13:28,123 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for af2a9e847c9e4d4812af432c436c7ac0: Running coprocessor pre-open hook at 1732018408104Writing region info on filesystem at 1732018408104Initializing all the Stores at 1732018408105 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408105Cleaning up temporary data from old regions at 1732018408114 (+9 ms)Running coprocessor post-open hooks at 1732018408123 (+9 ms)Region opened successfully at 1732018408123 2024-11-19T12:13:28,123 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 2ec97ffd6b2cc998be397eec7fc0a0bc 2024-11-19T12:13:28,125 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732018406961.af2a9e847c9e4d4812af432c436c7ac0., pid=44, masterSystemTime=1732018408033 2024-11-19T12:13:28,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8bfccd6e991fb7a90ce5bcadc0a3ef5, server=351a98347401,35543,1732018404609 because future has completed 2024-11-19T12:13:28,127 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 7382ad40044282a358da8130ce023367 2024-11-19T12:13:28,128 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,45367,1732018404555 2024-11-19T12:13:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T12:13:28,128 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 2ec97ffd6b2cc998be397eec7fc0a0bc 2024-11-19T12:13:28,129 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732018406961.af2a9e847c9e4d4812af432c436c7ac0. 2024-11-19T12:13:28,129 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,yyy,1732018406961.af2a9e847c9e4d4812af432c436c7ac0. 2024-11-19T12:13:28,129 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,fff,1732018406961.a78d1de57ec917279b938f869bdbe10d. 2024-11-19T12:13:28,129 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => a78d1de57ec917279b938f869bdbe10d, NAME => 'test,fff,1732018406961.a78d1de57ec917279b938f869bdbe10d.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-19T12:13:28,130 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a78d1de57ec917279b938f869bdbe10d 2024-11-19T12:13:28,130 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,fff,1732018406961.a78d1de57ec917279b938f869bdbe10d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,130 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for a78d1de57ec917279b938f869bdbe10d 2024-11-19T12:13:28,130 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for a78d1de57ec917279b938f869bdbe10d 2024-11-19T12:13:28,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b9b9ba699dcd89ef4b3398f2beb2455, server=351a98347401,45367,1732018404555 because future has completed 2024-11-19T12:13:28,134 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/2ec97ffd6b2cc998be397eec7fc0a0bc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,134 INFO [StoreOpener-a78d1de57ec917279b938f869bdbe10d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a78d1de57ec917279b938f869bdbe10d 2024-11-19T12:13:28,135 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 2ec97ffd6b2cc998be397eec7fc0a0bc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68738421, jitterRate=0.024282291531562805}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,136 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2ec97ffd6b2cc998be397eec7fc0a0bc 2024-11-19T12:13:28,136 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 2ec97ffd6b2cc998be397eec7fc0a0bc: Running coprocessor pre-open hook at 1732018408111Writing region info on filesystem at 1732018408111Initializing all the Stores at 1732018408113 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408113Cleaning up temporary data from old regions at 1732018408123 (+10 ms)Running coprocessor post-open hooks at 1732018408136 (+13 ms)Region opened successfully at 1732018408136 2024-11-19T12:13:28,136 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/7382ad40044282a358da8130ce023367/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,137 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:28,137 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 7382ad40044282a358da8130ce023367; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61132988, jitterRate=-0.08904749155044556}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,137 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7382ad40044282a358da8130ce023367 2024-11-19T12:13:28,137 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 7382ad40044282a358da8130ce023367: Running coprocessor pre-open hook at 1732018408114Writing region info on filesystem at 1732018408114Initializing all the Stores at 1732018408116 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408116Cleaning up temporary data from old regions at 1732018408123 (+7 ms)Running coprocessor post-open hooks at 1732018408137 (+14 ms)Region opened successfully at 1732018408137 2024-11-19T12:13:28,139 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732018406961.7382ad40044282a358da8130ce023367., pid=50, masterSystemTime=1732018408062 2024-11-19T12:13:28,139 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732018406961.2ec97ffd6b2cc998be397eec7fc0a0bc., pid=32, masterSystemTime=1732018407972 2024-11-19T12:13:28,140 INFO [StoreOpener-a78d1de57ec917279b938f869bdbe10d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a78d1de57ec917279b938f869bdbe10d columnFamilyName family 2024-11-19T12:13:28,140 DEBUG [StoreOpener-a78d1de57ec917279b938f869bdbe10d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,141 INFO [StoreOpener-a78d1de57ec917279b938f869bdbe10d-1 {}] regionserver.HStore(327): Store=a78d1de57ec917279b938f869bdbe10d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure af2a9e847c9e4d4812af432c436c7ac0, server=351a98347401,36275,1732018404438 because future has completed 2024-11-19T12:13:28,141 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for a78d1de57ec917279b938f869bdbe10d 2024-11-19T12:13:28,143 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/a78d1de57ec917279b938f869bdbe10d 2024-11-19T12:13:28,144 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/a78d1de57ec917279b938f869bdbe10d 2024-11-19T12:13:28,145 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for a78d1de57ec917279b938f869bdbe10d 2024-11-19T12:13:28,145 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for a78d1de57ec917279b938f869bdbe10d 2024-11-19T12:13:28,148 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732018406961.7382ad40044282a358da8130ce023367. 2024-11-19T12:13:28,148 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for a78d1de57ec917279b938f869bdbe10d 2024-11-19T12:13:28,148 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,hhh,1732018406961.7382ad40044282a358da8130ce023367. 2024-11-19T12:13:28,148 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571. 2024-11-19T12:13:28,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=7 2024-11-19T12:13:28,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 5b9b9ba699dcd89ef4b3398f2beb2455, server=351a98347401,45367,1732018404555 in 269 msec 2024-11-19T12:13:28,149 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732018406961.2ec97ffd6b2cc998be397eec7fc0a0bc. 2024-11-19T12:13:28,149 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,ttt,1732018406961.2ec97ffd6b2cc998be397eec7fc0a0bc. 2024-11-19T12:13:28,149 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,nnn,1732018406961.ef4298fd6621c8e7facf42422bd80be4. 2024-11-19T12:13:28,150 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => ef4298fd6621c8e7facf42422bd80be4, NAME => 'test,nnn,1732018406961.ef4298fd6621c8e7facf42422bd80be4.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-19T12:13:28,150 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ef4298fd6621c8e7facf42422bd80be4 2024-11-19T12:13:28,150 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,nnn,1732018406961.ef4298fd6621c8e7facf42422bd80be4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,150 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for ef4298fd6621c8e7facf42422bd80be4 2024-11-19T12:13:28,150 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for ef4298fd6621c8e7facf42422bd80be4 2024-11-19T12:13:28,149 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 11d894b29c3ee7cc1e5281c570908571, NAME => 'test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-19T12:13:28,151 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:28,152 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,152 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,45367,1732018404555 2024-11-19T12:13:28,152 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:28,152 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:28,153 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/a78d1de57ec917279b938f869bdbe10d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ec97ffd6b2cc998be397eec7fc0a0bc, server=351a98347401,45367,1732018404555 because future has completed 2024-11-19T12:13:28,155 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened a78d1de57ec917279b938f869bdbe10d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72476740, jitterRate=0.07998758554458618}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,155 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a78d1de57ec917279b938f869bdbe10d 2024-11-19T12:13:28,156 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for a78d1de57ec917279b938f869bdbe10d: Running coprocessor pre-open hook at 1732018408130Writing region info on filesystem at 1732018408130Initializing all the Stores at 1732018408133 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408133Cleaning up temporary data from old regions at 1732018408145 (+12 ms)Running coprocessor post-open hooks at 1732018408155 (+10 ms)Region opened successfully at 1732018408155 2024-11-19T12:13:28,156 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732018406961.a78d1de57ec917279b938f869bdbe10d., pid=48, masterSystemTime=1732018408033 2024-11-19T12:13:28,157 INFO [StoreOpener-11d894b29c3ee7cc1e5281c570908571-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:28,157 INFO [StoreOpener-ef4298fd6621c8e7facf42422bd80be4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ef4298fd6621c8e7facf42422bd80be4 2024-11-19T12:13:28,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5b9b9ba699dcd89ef4b3398f2beb2455, ASSIGN in 515 msec 2024-11-19T12:13:28,160 INFO [StoreOpener-11d894b29c3ee7cc1e5281c570908571-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11d894b29c3ee7cc1e5281c570908571 columnFamilyName family 2024-11-19T12:13:28,160 DEBUG [StoreOpener-11d894b29c3ee7cc1e5281c570908571-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,160 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732018406961.a78d1de57ec917279b938f869bdbe10d. 2024-11-19T12:13:28,160 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,fff,1732018406961.a78d1de57ec917279b938f869bdbe10d. 2024-11-19T12:13:28,160 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,vvv,1732018406961.2b6d3f4b598ab015d9c62b67f0bac50f. 2024-11-19T12:13:28,160 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 2b6d3f4b598ab015d9c62b67f0bac50f, NAME => 'test,vvv,1732018406961.2b6d3f4b598ab015d9c62b67f0bac50f.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-19T12:13:28,161 INFO [StoreOpener-ef4298fd6621c8e7facf42422bd80be4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef4298fd6621c8e7facf42422bd80be4 columnFamilyName family 2024-11-19T12:13:28,161 DEBUG [StoreOpener-ef4298fd6621c8e7facf42422bd80be4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,161 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2b6d3f4b598ab015d9c62b67f0bac50f 2024-11-19T12:13:28,161 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,vvv,1732018406961.2b6d3f4b598ab015d9c62b67f0bac50f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,162 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 2b6d3f4b598ab015d9c62b67f0bac50f 2024-11-19T12:13:28,162 INFO [StoreOpener-11d894b29c3ee7cc1e5281c570908571-1 {}] regionserver.HStore(327): Store=11d894b29c3ee7cc1e5281c570908571/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,162 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 2b6d3f4b598ab015d9c62b67f0bac50f 2024-11-19T12:13:28,162 INFO [StoreOpener-ef4298fd6621c8e7facf42422bd80be4-1 {}] regionserver.HStore(327): Store=ef4298fd6621c8e7facf42422bd80be4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,162 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:28,162 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for ef4298fd6621c8e7facf42422bd80be4 2024-11-19T12:13:28,163 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/ef4298fd6621c8e7facf42422bd80be4 2024-11-19T12:13:28,164 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:28,164 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/ef4298fd6621c8e7facf42422bd80be4 2024-11-19T12:13:28,165 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:28,165 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:28,166 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:28,166 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for ef4298fd6621c8e7facf42422bd80be4 2024-11-19T12:13:28,166 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for ef4298fd6621c8e7facf42422bd80be4 2024-11-19T12:13:28,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=24 2024-11-19T12:13:28,167 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=351a98347401,35543,1732018404609, table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T12:13:28,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 2ec97ffd6b2cc998be397eec7fc0a0bc, server=351a98347401,45367,1732018404555 in 335 msec 2024-11-19T12:13:28,168 INFO [StoreOpener-2b6d3f4b598ab015d9c62b67f0bac50f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2b6d3f4b598ab015d9c62b67f0bac50f 2024-11-19T12:13:28,169 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:28,169 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for ef4298fd6621c8e7facf42422bd80be4 2024-11-19T12:13:28,172 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/ef4298fd6621c8e7facf42422bd80be4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,173 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/11d894b29c3ee7cc1e5281c570908571/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,173 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened ef4298fd6621c8e7facf42422bd80be4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69776572, jitterRate=0.03975194692611694}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,173 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 11d894b29c3ee7cc1e5281c570908571; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59515882, jitterRate=-0.11314424872398376}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,173 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ef4298fd6621c8e7facf42422bd80be4 2024-11-19T12:13:28,173 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:28,173 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 11d894b29c3ee7cc1e5281c570908571: Running coprocessor pre-open hook at 1732018408152Writing region info on filesystem at 1732018408152Initializing all the Stores at 1732018408154 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408154Cleaning up temporary data from old regions at 1732018408166 (+12 ms)Running coprocessor post-open hooks at 1732018408173 (+7 ms)Region opened successfully at 1732018408173 2024-11-19T12:13:28,173 INFO [StoreOpener-2b6d3f4b598ab015d9c62b67f0bac50f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b6d3f4b598ab015d9c62b67f0bac50f columnFamilyName family 2024-11-19T12:13:28,173 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for ef4298fd6621c8e7facf42422bd80be4: Running coprocessor pre-open hook at 1732018408150Writing region info on filesystem at 1732018408150Initializing all the Stores at 1732018408152 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408152Cleaning up temporary data from old regions at 1732018408166 (+14 ms)Running coprocessor post-open hooks at 1732018408173 (+7 ms)Region opened successfully at 1732018408173 2024-11-19T12:13:28,174 DEBUG [StoreOpener-2b6d3f4b598ab015d9c62b67f0bac50f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,174 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732018406961.ef4298fd6621c8e7facf42422bd80be4., pid=36, masterSystemTime=1732018407972 2024-11-19T12:13:28,175 INFO [StoreOpener-2b6d3f4b598ab015d9c62b67f0bac50f-1 {}] regionserver.HStore(327): Store=2b6d3f4b598ab015d9c62b67f0bac50f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,175 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571., pid=46, masterSystemTime=1732018408062 2024-11-19T12:13:28,176 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 2b6d3f4b598ab015d9c62b67f0bac50f 2024-11-19T12:13:28,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2ec97ffd6b2cc998be397eec7fc0a0bc, ASSIGN in 528 msec 2024-11-19T12:13:28,178 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/2b6d3f4b598ab015d9c62b67f0bac50f 2024-11-19T12:13:28,178 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732018406961.ef4298fd6621c8e7facf42422bd80be4. 2024-11-19T12:13:28,178 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,nnn,1732018406961.ef4298fd6621c8e7facf42422bd80be4. 2024-11-19T12:13:28,178 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,kkk,1732018406961.a329354a61adc5325fa166bf564b84eb. 2024-11-19T12:13:28,178 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:28,178 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => a329354a61adc5325fa166bf564b84eb, NAME => 'test,kkk,1732018406961.a329354a61adc5325fa166bf564b84eb.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-19T12:13:28,178 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/2b6d3f4b598ab015d9c62b67f0bac50f 2024-11-19T12:13:28,179 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a329354a61adc5325fa166bf564b84eb 2024-11-19T12:13:28,179 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,kkk,1732018406961.a329354a61adc5325fa166bf564b84eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,179 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for a329354a61adc5325fa166bf564b84eb 2024-11-19T12:13:28,179 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for a329354a61adc5325fa166bf564b84eb 2024-11-19T12:13:28,179 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 2b6d3f4b598ab015d9c62b67f0bac50f 2024-11-19T12:13:28,179 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 2b6d3f4b598ab015d9c62b67f0bac50f 2024-11-19T12:13:28,181 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571. 2024-11-19T12:13:28,181 INFO [StoreOpener-a329354a61adc5325fa166bf564b84eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a329354a61adc5325fa166bf564b84eb 2024-11-19T12:13:28,182 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571. 2024-11-19T12:13:28,182 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3. 2024-11-19T12:13:28,182 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, NAME => 'test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-19T12:13:28,182 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:28,183 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,183 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:28,183 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:28,183 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure a78d1de57ec917279b938f869bdbe10d, server=351a98347401,36275,1732018404438 because future has completed 2024-11-19T12:13:28,184 INFO [StoreOpener-a329354a61adc5325fa166bf564b84eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a329354a61adc5325fa166bf564b84eb columnFamilyName family 2024-11-19T12:13:28,184 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=11d894b29c3ee7cc1e5281c570908571, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,35543,1732018404609 2024-11-19T12:13:28,184 DEBUG [StoreOpener-a329354a61adc5325fa166bf564b84eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,185 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 2b6d3f4b598ab015d9c62b67f0bac50f 2024-11-19T12:13:28,185 INFO [StoreOpener-c48a9e94bd9fcbd6f8c1f6ae2b6c80d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:28,185 INFO [StoreOpener-a329354a61adc5325fa166bf564b84eb-1 {}] regionserver.HStore(327): Store=a329354a61adc5325fa166bf564b84eb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,185 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for a329354a61adc5325fa166bf564b84eb 2024-11-19T12:13:28,187 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/a329354a61adc5325fa166bf564b84eb 2024-11-19T12:13:28,187 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/a329354a61adc5325fa166bf564b84eb 2024-11-19T12:13:28,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11d894b29c3ee7cc1e5281c570908571, server=351a98347401,35543,1732018404609 because future has completed 2024-11-19T12:13:28,188 INFO [StoreOpener-c48a9e94bd9fcbd6f8c1f6ae2b6c80d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 columnFamilyName family 2024-11-19T12:13:28,189 DEBUG [StoreOpener-c48a9e94bd9fcbd6f8c1f6ae2b6c80d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,189 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for a329354a61adc5325fa166bf564b84eb 2024-11-19T12:13:28,189 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for a329354a61adc5325fa166bf564b84eb 2024-11-19T12:13:28,190 INFO [StoreOpener-c48a9e94bd9fcbd6f8c1f6ae2b6c80d3-1 {}] regionserver.HStore(327): Store=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,190 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:28,191 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/2b6d3f4b598ab015d9c62b67f0bac50f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,191 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:28,192 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:28,192 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for a329354a61adc5325fa166bf564b84eb 2024-11-19T12:13:28,192 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 2b6d3f4b598ab015d9c62b67f0bac50f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68598016, jitterRate=0.022190093994140625}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,192 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2b6d3f4b598ab015d9c62b67f0bac50f 2024-11-19T12:13:28,192 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 2b6d3f4b598ab015d9c62b67f0bac50f: Running coprocessor pre-open hook at 1732018408163Writing region info on filesystem at 1732018408163Initializing all the Stores at 1732018408166 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408166Cleaning up temporary data from old regions at 1732018408179 (+13 ms)Running coprocessor post-open hooks at 1732018408192 (+13 ms)Region opened successfully at 1732018408192 2024-11-19T12:13:28,192 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:28,192 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:28,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=10 2024-11-19T12:13:28,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure a78d1de57ec917279b938f869bdbe10d, server=351a98347401,36275,1732018404438 in 271 msec 2024-11-19T12:13:28,194 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732018406961.2b6d3f4b598ab015d9c62b67f0bac50f., pid=43, masterSystemTime=1732018408033 2024-11-19T12:13:28,195 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:28,195 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/a329354a61adc5325fa166bf564b84eb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,196 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened a329354a61adc5325fa166bf564b84eb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62628759, jitterRate=-0.06675876677036285}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,196 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a329354a61adc5325fa166bf564b84eb 2024-11-19T12:13:28,196 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for a329354a61adc5325fa166bf564b84eb: Running coprocessor pre-open hook at 1732018408179Writing region info on filesystem at 1732018408179Initializing all the Stores at 1732018408181 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408181Cleaning up temporary data from old regions at 1732018408189 (+8 ms)Running coprocessor post-open hooks at 1732018408196 (+7 ms)Region opened successfully at 1732018408196 2024-11-19T12:13:28,198 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732018406961.a329354a61adc5325fa166bf564b84eb., pid=33, masterSystemTime=1732018407972 2024-11-19T12:13:28,199 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/c48a9e94bd9fcbd6f8c1f6ae2b6c80d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,200 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened c48a9e94bd9fcbd6f8c1f6ae2b6c80d3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62126062, jitterRate=-0.07424953579902649}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,200 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:28,200 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for c48a9e94bd9fcbd6f8c1f6ae2b6c80d3: Running coprocessor pre-open hook at 1732018408183Writing region info on filesystem at 1732018408183Initializing all the Stores at 1732018408184 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408184Cleaning up temporary data from old regions at 1732018408192 (+8 ms)Running coprocessor post-open hooks at 1732018408200 (+8 ms)Region opened successfully at 1732018408200 2024-11-19T12:13:28,201 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732018406961.2b6d3f4b598ab015d9c62b67f0bac50f. 2024-11-19T12:13:28,201 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3., pid=49, masterSystemTime=1732018408062 2024-11-19T12:13:28,201 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,vvv,1732018406961.2b6d3f4b598ab015d9c62b67f0bac50f. 2024-11-19T12:13:28,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a78d1de57ec917279b938f869bdbe10d, ASSIGN in 554 msec 2024-11-19T12:13:28,202 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,jjj,1732018406961.19add541d34392b48fdd488f1c19c182. 2024-11-19T12:13:28,202 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732018406961.a329354a61adc5325fa166bf564b84eb. 2024-11-19T12:13:28,202 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 19add541d34392b48fdd488f1c19c182, NAME => 'test,jjj,1732018406961.19add541d34392b48fdd488f1c19c182.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-19T12:13:28,202 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,kkk,1732018406961.a329354a61adc5325fa166bf564b84eb. 2024-11-19T12:13:28,202 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,zzz,1732018406961.4fea5432239dac4bfb1b607057a95e34. 2024-11-19T12:13:28,203 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 19add541d34392b48fdd488f1c19c182 2024-11-19T12:13:28,203 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 4fea5432239dac4bfb1b607057a95e34, NAME => 'test,zzz,1732018406961.4fea5432239dac4bfb1b607057a95e34.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-19T12:13:28,203 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,jjj,1732018406961.19add541d34392b48fdd488f1c19c182.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,203 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 19add541d34392b48fdd488f1c19c182 2024-11-19T12:13:28,203 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 19add541d34392b48fdd488f1c19c182 2024-11-19T12:13:28,203 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4fea5432239dac4bfb1b607057a95e34 2024-11-19T12:13:28,203 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,zzz,1732018406961.4fea5432239dac4bfb1b607057a95e34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,203 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 4fea5432239dac4bfb1b607057a95e34 2024-11-19T12:13:28,203 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 4fea5432239dac4bfb1b607057a95e34 2024-11-19T12:13:28,205 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,45367,1732018404555 2024-11-19T12:13:28,205 INFO [StoreOpener-19add541d34392b48fdd488f1c19c182-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 19add541d34392b48fdd488f1c19c182 2024-11-19T12:13:28,205 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3. 2024-11-19T12:13:28,205 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3. 2024-11-19T12:13:28,206 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,,1732018406961.b859464d4ffda6f2df23654fb949364f. 2024-11-19T12:13:28,206 INFO [StoreOpener-4fea5432239dac4bfb1b607057a95e34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4fea5432239dac4bfb1b607057a95e34 2024-11-19T12:13:28,206 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => b859464d4ffda6f2df23654fb949364f, NAME => 'test,,1732018406961.b859464d4ffda6f2df23654fb949364f.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-19T12:13:28,206 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:28,206 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,,1732018406961.b859464d4ffda6f2df23654fb949364f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,206 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:28,207 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:28,208 INFO [StoreOpener-19add541d34392b48fdd488f1c19c182-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19add541d34392b48fdd488f1c19c182 columnFamilyName family 2024-11-19T12:13:28,208 INFO [StoreOpener-4fea5432239dac4bfb1b607057a95e34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4fea5432239dac4bfb1b607057a95e34 columnFamilyName family 2024-11-19T12:13:28,208 DEBUG [StoreOpener-4fea5432239dac4bfb1b607057a95e34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,208 DEBUG [StoreOpener-19add541d34392b48fdd488f1c19c182-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure a329354a61adc5325fa166bf564b84eb, server=351a98347401,45367,1732018404555 because future has completed 2024-11-19T12:13:28,209 INFO [StoreOpener-4fea5432239dac4bfb1b607057a95e34-1 {}] regionserver.HStore(327): Store=4fea5432239dac4bfb1b607057a95e34/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,209 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 4fea5432239dac4bfb1b607057a95e34 2024-11-19T12:13:28,210 INFO [StoreOpener-19add541d34392b48fdd488f1c19c182-1 {}] regionserver.HStore(327): Store=19add541d34392b48fdd488f1c19c182/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,211 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 19add541d34392b48fdd488f1c19c182 2024-11-19T12:13:28,211 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/4fea5432239dac4bfb1b607057a95e34 2024-11-19T12:13:28,211 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/4fea5432239dac4bfb1b607057a95e34 2024-11-19T12:13:28,211 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/19add541d34392b48fdd488f1c19c182 2024-11-19T12:13:28,212 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,35543,1732018404609 2024-11-19T12:13:28,212 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 4fea5432239dac4bfb1b607057a95e34 2024-11-19T12:13:28,212 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/19add541d34392b48fdd488f1c19c182 2024-11-19T12:13:28,212 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 4fea5432239dac4bfb1b607057a95e34 2024-11-19T12:13:28,212 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 19add541d34392b48fdd488f1c19c182 2024-11-19T12:13:28,212 INFO [StoreOpener-b859464d4ffda6f2df23654fb949364f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:28,213 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 19add541d34392b48fdd488f1c19c182 2024-11-19T12:13:28,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, server=351a98347401,35543,1732018404609 because future has completed 2024-11-19T12:13:28,215 INFO [StoreOpener-b859464d4ffda6f2df23654fb949364f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b859464d4ffda6f2df23654fb949364f columnFamilyName family 2024-11-19T12:13:28,216 DEBUG [StoreOpener-b859464d4ffda6f2df23654fb949364f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,216 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 19add541d34392b48fdd488f1c19c182 2024-11-19T12:13:28,216 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 4fea5432239dac4bfb1b607057a95e34 2024-11-19T12:13:28,217 INFO [StoreOpener-b859464d4ffda6f2df23654fb949364f-1 {}] regionserver.HStore(327): Store=b859464d4ffda6f2df23654fb949364f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,217 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:28,219 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:28,219 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:28,220 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:28,220 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:28,221 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/19add541d34392b48fdd488f1c19c182/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=15 2024-11-19T12:13:28,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure a329354a61adc5325fa166bf564b84eb, server=351a98347401,45367,1732018404555 in 380 msec 2024-11-19T12:13:28,222 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 19add541d34392b48fdd488f1c19c182; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69226588, jitterRate=0.031556546688079834}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,222 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 19add541d34392b48fdd488f1c19c182 2024-11-19T12:13:28,222 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 19add541d34392b48fdd488f1c19c182: Running coprocessor pre-open hook at 1732018408203Writing region info on filesystem at 1732018408203Initializing all the Stores at 1732018408205 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408205Cleaning up temporary data from old regions at 1732018408213 (+8 ms)Running coprocessor post-open hooks at 1732018408222 (+9 ms)Region opened successfully at 1732018408222 2024-11-19T12:13:28,223 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:28,224 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732018406961.19add541d34392b48fdd488f1c19c182., pid=45, masterSystemTime=1732018408033 2024-11-19T12:13:28,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a329354a61adc5325fa166bf564b84eb, ASSIGN in 581 msec 2024-11-19T12:13:28,227 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/4fea5432239dac4bfb1b607057a95e34/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,227 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732018406961.19add541d34392b48fdd488f1c19c182. 2024-11-19T12:13:28,227 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,jjj,1732018406961.19add541d34392b48fdd488f1c19c182. 2024-11-19T12:13:28,228 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,sss,1732018406961.797a72fe833580fc974f887653d7763a. 2024-11-19T12:13:28,228 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 797a72fe833580fc974f887653d7763a, NAME => 'test,sss,1732018406961.797a72fe833580fc974f887653d7763a.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-19T12:13:28,228 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 4fea5432239dac4bfb1b607057a95e34; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68051887, jitterRate=0.014052137732505798}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,228 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4fea5432239dac4bfb1b607057a95e34 2024-11-19T12:13:28,228 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 4fea5432239dac4bfb1b607057a95e34: Running coprocessor pre-open hook at 1732018408204Writing region info on filesystem at 1732018408204Initializing all the Stores at 1732018408205 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408205Cleaning up temporary data from old regions at 1732018408212 (+7 ms)Running coprocessor post-open hooks at 1732018408228 (+16 ms)Region opened successfully at 1732018408228 2024-11-19T12:13:28,228 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 797a72fe833580fc974f887653d7763a 2024-11-19T12:13:28,228 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,sss,1732018406961.797a72fe833580fc974f887653d7763a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,228 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/b859464d4ffda6f2df23654fb949364f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,229 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 797a72fe833580fc974f887653d7763a 2024-11-19T12:13:28,229 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 797a72fe833580fc974f887653d7763a 2024-11-19T12:13:28,229 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732018406961.4fea5432239dac4bfb1b607057a95e34., pid=31, masterSystemTime=1732018407972 2024-11-19T12:13:28,229 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened b859464d4ffda6f2df23654fb949364f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71942477, jitterRate=0.07202644646167755}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,230 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:28,230 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for b859464d4ffda6f2df23654fb949364f: Running coprocessor pre-open hook at 1732018408207Writing region info on filesystem at 1732018408207Initializing all the Stores at 1732018408209 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408209Cleaning up temporary data from old regions at 1732018408220 (+11 ms)Running coprocessor post-open hooks at 1732018408230 (+10 ms)Region opened successfully at 1732018408230 2024-11-19T12:13:28,232 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732018406961.b859464d4ffda6f2df23654fb949364f., pid=56, masterSystemTime=1732018408062 2024-11-19T12:13:28,233 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732018406961.4fea5432239dac4bfb1b607057a95e34. 2024-11-19T12:13:28,233 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,zzz,1732018406961.4fea5432239dac4bfb1b607057a95e34. 2024-11-19T12:13:28,233 INFO [StoreOpener-797a72fe833580fc974f887653d7763a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 797a72fe833580fc974f887653d7763a 2024-11-19T12:13:28,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=25 2024-11-19T12:13:28,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, server=351a98347401,35543,1732018404609 in 301 msec 2024-11-19T12:13:28,235 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732018406961.b859464d4ffda6f2df23654fb949364f. 2024-11-19T12:13:28,235 INFO [StoreOpener-797a72fe833580fc974f887653d7763a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 797a72fe833580fc974f887653d7763a columnFamilyName family 2024-11-19T12:13:28,235 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,,1732018406961.b859464d4ffda6f2df23654fb949364f. 2024-11-19T12:13:28,236 DEBUG [StoreOpener-797a72fe833580fc974f887653d7763a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,236 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,iii,1732018406961.ab050043367e9fb30539590d5f965d47. 2024-11-19T12:13:28,236 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => ab050043367e9fb30539590d5f965d47, NAME => 'test,iii,1732018406961.ab050043367e9fb30539590d5f965d47.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-19T12:13:28,236 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:28,236 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,iii,1732018406961.ab050043367e9fb30539590d5f965d47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,237 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:28,237 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:28,237 INFO [StoreOpener-797a72fe833580fc974f887653d7763a-1 {}] regionserver.HStore(327): Store=797a72fe833580fc974f887653d7763a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,237 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 797a72fe833580fc974f887653d7763a 2024-11-19T12:13:28,238 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/797a72fe833580fc974f887653d7763a 2024-11-19T12:13:28,239 INFO [StoreOpener-ab050043367e9fb30539590d5f965d47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:28,239 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/797a72fe833580fc974f887653d7763a 2024-11-19T12:13:28,239 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 797a72fe833580fc974f887653d7763a 2024-11-19T12:13:28,239 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 797a72fe833580fc974f887653d7763a 2024-11-19T12:13:28,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN in 595 msec 2024-11-19T12:13:28,241 INFO [StoreOpener-ab050043367e9fb30539590d5f965d47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab050043367e9fb30539590d5f965d47 columnFamilyName family 2024-11-19T12:13:28,241 DEBUG [StoreOpener-ab050043367e9fb30539590d5f965d47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,242 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 797a72fe833580fc974f887653d7763a 2024-11-19T12:13:28,242 INFO [StoreOpener-ab050043367e9fb30539590d5f965d47-1 {}] regionserver.HStore(327): Store=ab050043367e9fb30539590d5f965d47/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,243 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:28,243 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b859464d4ffda6f2df23654fb949364f, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,35543,1732018404609 2024-11-19T12:13:28,244 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:28,247 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:28,247 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=351a98347401,36275,1732018404438, table=test, region=af2a9e847c9e4d4812af432c436c7ac0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T12:13:28,248 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,45367,1732018404555 2024-11-19T12:13:28,248 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:28,249 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:28,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,35543,1732018404609 because future has completed 2024-11-19T12:13:28,249 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/797a72fe833580fc974f887653d7763a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,250 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 797a72fe833580fc974f887653d7763a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71582096, jitterRate=0.06665635108947754}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,250 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 797a72fe833580fc974f887653d7763a 2024-11-19T12:13:28,250 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 797a72fe833580fc974f887653d7763a: Running coprocessor pre-open hook at 1732018408229Writing region info on filesystem at 1732018408229Initializing all the Stores at 1732018408230 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408231 (+1 ms)Cleaning up temporary data from old regions at 1732018408239 (+8 ms)Running coprocessor post-open hooks at 1732018408250 (+11 ms)Region opened successfully at 1732018408250 2024-11-19T12:13:28,251 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732018406961.797a72fe833580fc974f887653d7763a., pid=40, masterSystemTime=1732018408033 2024-11-19T12:13:28,252 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:28,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4fea5432239dac4bfb1b607057a95e34, server=351a98347401,45367,1732018404555 because future has completed 2024-11-19T12:13:28,256 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732018406961.797a72fe833580fc974f887653d7763a. 2024-11-19T12:13:28,256 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,sss,1732018406961.797a72fe833580fc974f887653d7763a. 2024-11-19T12:13:28,256 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,bbb,1732018406961.e257775d8750f12b70161bb0b6387163. 2024-11-19T12:13:28,256 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=351a98347401,45367,1732018404555, table=test, region=4fea5432239dac4bfb1b607057a95e34. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T12:13:28,256 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => e257775d8750f12b70161bb0b6387163, NAME => 'test,bbb,1732018406961.e257775d8750f12b70161bb0b6387163.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-19T12:13:28,257 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/ab050043367e9fb30539590d5f965d47/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,257 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e257775d8750f12b70161bb0b6387163 2024-11-19T12:13:28,257 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,bbb,1732018406961.e257775d8750f12b70161bb0b6387163.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,257 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for e257775d8750f12b70161bb0b6387163 2024-11-19T12:13:28,257 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for e257775d8750f12b70161bb0b6387163 2024-11-19T12:13:28,258 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened ab050043367e9fb30539590d5f965d47; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72340828, jitterRate=0.07796233892440796}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,258 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:28,258 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for ab050043367e9fb30539590d5f965d47: Running coprocessor pre-open hook at 1732018408237Writing region info on filesystem at 1732018408237Initializing all the Stores at 1732018408238 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408238Cleaning up temporary data from old regions at 1732018408249 (+11 ms)Running coprocessor post-open hooks at 1732018408258 (+9 ms)Region opened successfully at 1732018408258 2024-11-19T12:13:28,259 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732018406961.ab050043367e9fb30539590d5f965d47., pid=51, masterSystemTime=1732018408062 2024-11-19T12:13:28,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=5 2024-11-19T12:13:28,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,35543,1732018404609 in 289 msec 2024-11-19T12:13:28,260 INFO [StoreOpener-e257775d8750f12b70161bb0b6387163-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e257775d8750f12b70161bb0b6387163 2024-11-19T12:13:28,262 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732018406961.ab050043367e9fb30539590d5f965d47. 2024-11-19T12:13:28,262 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,iii,1732018406961.ab050043367e9fb30539590d5f965d47. 2024-11-19T12:13:28,262 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff. 2024-11-19T12:13:28,263 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => f9224ed53fa7acfead92d4143c8626ff, NAME => 'test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-19T12:13:28,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN in 620 msec 2024-11-19T12:13:28,263 INFO [StoreOpener-e257775d8750f12b70161bb0b6387163-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e257775d8750f12b70161bb0b6387163 columnFamilyName family 2024-11-19T12:13:28,263 DEBUG [StoreOpener-e257775d8750f12b70161bb0b6387163-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,263 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:28,263 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,264 INFO [StoreOpener-e257775d8750f12b70161bb0b6387163-1 {}] regionserver.HStore(327): Store=e257775d8750f12b70161bb0b6387163/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,264 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:28,264 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:28,264 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for e257775d8750f12b70161bb0b6387163 2024-11-19T12:13:28,265 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=ab050043367e9fb30539590d5f965d47, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,35543,1732018404609 2024-11-19T12:13:28,265 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/e257775d8750f12b70161bb0b6387163 2024-11-19T12:13:28,265 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/e257775d8750f12b70161bb0b6387163 2024-11-19T12:13:28,266 INFO [StoreOpener-f9224ed53fa7acfead92d4143c8626ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:28,266 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for e257775d8750f12b70161bb0b6387163 2024-11-19T12:13:28,266 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for e257775d8750f12b70161bb0b6387163 2024-11-19T12:13:28,268 INFO [StoreOpener-f9224ed53fa7acfead92d4143c8626ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9224ed53fa7acfead92d4143c8626ff columnFamilyName family 2024-11-19T12:13:28,268 DEBUG [StoreOpener-f9224ed53fa7acfead92d4143c8626ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab050043367e9fb30539590d5f965d47, server=351a98347401,35543,1732018404609 because future has completed 2024-11-19T12:13:28,269 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for e257775d8750f12b70161bb0b6387163 2024-11-19T12:13:28,269 INFO [StoreOpener-f9224ed53fa7acfead92d4143c8626ff-1 {}] regionserver.HStore(327): Store=f9224ed53fa7acfead92d4143c8626ff/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,269 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:28,270 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=797a72fe833580fc974f887653d7763a, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:28,272 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:28,272 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=351a98347401,35543,1732018404609, table=test, region=ab050043367e9fb30539590d5f965d47. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T12:13:28,272 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=351a98347401,35543,1732018404609, table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T12:13:28,272 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=351a98347401,35543,1732018404609, table=test, region=11d894b29c3ee7cc1e5281c570908571. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T12:13:28,274 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:28,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 797a72fe833580fc974f887653d7763a, server=351a98347401,36275,1732018404438 because future has completed 2024-11-19T12:13:28,274 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/e257775d8750f12b70161bb0b6387163/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,275 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:28,275 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:28,275 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened e257775d8750f12b70161bb0b6387163; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68288665, jitterRate=0.01758040487766266}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,275 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e257775d8750f12b70161bb0b6387163 2024-11-19T12:13:28,275 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for e257775d8750f12b70161bb0b6387163: Running coprocessor pre-open hook at 1732018408258Writing region info on filesystem at 1732018408258Initializing all the Stores at 1732018408259 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408259Cleaning up temporary data from old regions at 1732018408266 (+7 ms)Running coprocessor post-open hooks at 1732018408275 (+9 ms)Region opened successfully at 1732018408275 2024-11-19T12:13:28,276 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732018406961.e257775d8750f12b70161bb0b6387163., pid=47, masterSystemTime=1732018408033 2024-11-19T12:13:28,279 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:28,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=13 2024-11-19T12:13:28,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure ab050043367e9fb30539590d5f965d47, server=351a98347401,35543,1732018404609 in 335 msec 2024-11-19T12:13:28,280 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732018406961.e257775d8750f12b70161bb0b6387163. 2024-11-19T12:13:28,280 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,bbb,1732018406961.e257775d8750f12b70161bb0b6387163. 2024-11-19T12:13:28,281 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=e257775d8750f12b70161bb0b6387163, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:28,283 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/f9224ed53fa7acfead92d4143c8626ff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure e257775d8750f12b70161bb0b6387163, server=351a98347401,36275,1732018404438 because future has completed 2024-11-19T12:13:28,284 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened f9224ed53fa7acfead92d4143c8626ff; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63881828, jitterRate=-0.048086583614349365}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,284 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:28,285 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for f9224ed53fa7acfead92d4143c8626ff: Running coprocessor pre-open hook at 1732018408264Writing region info on filesystem at 1732018408264Initializing all the Stores at 1732018408265 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408265Cleaning up temporary data from old regions at 1732018408275 (+10 ms)Running coprocessor post-open hooks at 1732018408284 (+9 ms)Region opened successfully at 1732018408284 2024-11-19T12:13:28,286 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff., pid=55, masterSystemTime=1732018408062 2024-11-19T12:13:28,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN in 643 msec 2024-11-19T12:13:28,288 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff. 2024-11-19T12:13:28,288 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff. 2024-11-19T12:13:28,288 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e. 2024-11-19T12:13:28,288 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 5bc8ba883b6bd714e90daec2cb61209e, NAME => 'test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-19T12:13:28,289 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:28,289 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,289 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:28,289 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:28,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=6 2024-11-19T12:13:28,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure e257775d8750f12b70161bb0b6387163, server=351a98347401,36275,1732018404438 in 373 msec 2024-11-19T12:13:28,291 INFO [StoreOpener-5bc8ba883b6bd714e90daec2cb61209e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:28,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e257775d8750f12b70161bb0b6387163, ASSIGN in 651 msec 2024-11-19T12:13:28,293 INFO [StoreOpener-5bc8ba883b6bd714e90daec2cb61209e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bc8ba883b6bd714e90daec2cb61209e columnFamilyName family 2024-11-19T12:13:28,294 DEBUG [StoreOpener-5bc8ba883b6bd714e90daec2cb61209e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,294 INFO [StoreOpener-5bc8ba883b6bd714e90daec2cb61209e-1 {}] regionserver.HStore(327): Store=5bc8ba883b6bd714e90daec2cb61209e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,295 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:28,295 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=f9224ed53fa7acfead92d4143c8626ff, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,35543,1732018404609 2024-11-19T12:13:28,296 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:28,296 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:28,298 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:28,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9224ed53fa7acfead92d4143c8626ff, server=351a98347401,35543,1732018404609 because future has completed 2024-11-19T12:13:28,298 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:28,301 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:28,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=23 2024-11-19T12:13:28,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 797a72fe833580fc974f887653d7763a, server=351a98347401,36275,1732018404438 in 420 msec 2024-11-19T12:13:28,305 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/5bc8ba883b6bd714e90daec2cb61209e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=797a72fe833580fc974f887653d7763a, ASSIGN in 662 msec 2024-11-19T12:13:28,306 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 5bc8ba883b6bd714e90daec2cb61209e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62287936, jitterRate=-0.0718374252319336}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,306 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:28,306 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 5bc8ba883b6bd714e90daec2cb61209e: Running coprocessor pre-open hook at 1732018408289Writing region info on filesystem at 1732018408289Initializing all the Stores at 1732018408291 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408291Cleaning up temporary data from old regions at 1732018408298 (+7 ms)Running coprocessor post-open hooks at 1732018408306 (+8 ms)Region opened successfully at 1732018408306 2024-11-19T12:13:28,307 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e., pid=52, masterSystemTime=1732018408062 2024-11-19T12:13:28,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=9 2024-11-19T12:13:28,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure f9224ed53fa7acfead92d4143c8626ff, server=351a98347401,35543,1732018404609 in 346 msec 2024-11-19T12:13:28,310 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e. 2024-11-19T12:13:28,310 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e. 2024-11-19T12:13:28,310 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115. 2024-11-19T12:13:28,311 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 24b360a2b049dbc9c784cdd687321115, NAME => 'test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-19T12:13:28,311 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:28,311 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:28,311 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:28,311 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:28,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN in 670 msec 2024-11-19T12:13:28,314 INFO [StoreOpener-24b360a2b049dbc9c784cdd687321115-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:28,315 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,35543,1732018404609 2024-11-19T12:13:28,316 INFO [StoreOpener-24b360a2b049dbc9c784cdd687321115-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24b360a2b049dbc9c784cdd687321115 columnFamilyName family 2024-11-19T12:13:28,317 DEBUG [StoreOpener-24b360a2b049dbc9c784cdd687321115-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:28,317 INFO [StoreOpener-24b360a2b049dbc9c784cdd687321115-1 {}] regionserver.HStore(327): Store=24b360a2b049dbc9c784cdd687321115/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:28,318 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:28,319 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:28,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bc8ba883b6bd714e90daec2cb61209e, server=351a98347401,35543,1732018404609 because future has completed 2024-11-19T12:13:28,320 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:28,320 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:28,320 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:28,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-19T12:13:28,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 4fea5432239dac4bfb1b607057a95e34, server=351a98347401,45367,1732018404555 in 500 msec 2024-11-19T12:13:28,323 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:28,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4fea5432239dac4bfb1b607057a95e34, ASSIGN in 682 msec 2024-11-19T12:13:28,327 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/24b360a2b049dbc9c784cdd687321115/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:13:28,328 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 24b360a2b049dbc9c784cdd687321115; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67380682, jitterRate=0.004050403833389282}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:28,328 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:28,328 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 24b360a2b049dbc9c784cdd687321115: Running coprocessor pre-open hook at 1732018408312Writing region info on filesystem at 1732018408312Initializing all the Stores at 1732018408313 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018408313Cleaning up temporary data from old regions at 1732018408321 (+8 ms)Running coprocessor post-open hooks at 1732018408328 (+7 ms)Region opened successfully at 1732018408328 2024-11-19T12:13:28,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=16 2024-11-19T12:13:28,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 5bc8ba883b6bd714e90daec2cb61209e, server=351a98347401,35543,1732018404609 in 381 msec 2024-11-19T12:13:28,329 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115., pid=54, masterSystemTime=1732018408062 2024-11-19T12:13:28,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN in 689 msec 2024-11-19T12:13:28,332 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115. 2024-11-19T12:13:28,332 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115. 2024-11-19T12:13:28,333 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=24b360a2b049dbc9c784cdd687321115, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,35543,1732018404609 2024-11-19T12:13:28,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24b360a2b049dbc9c784cdd687321115, server=351a98347401,35543,1732018404609 because future has completed 2024-11-19T12:13:28,337 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=19add541d34392b48fdd488f1c19c182, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:28,340 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 19add541d34392b48fdd488f1c19c182, server=351a98347401,36275,1732018404438 because future has completed 2024-11-19T12:13:28,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=22 2024-11-19T12:13:28,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 24b360a2b049dbc9c784cdd687321115, server=351a98347401,35543,1732018404609 in 387 msec 2024-11-19T12:13:28,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN in 703 msec 2024-11-19T12:13:28,352 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=351a98347401,36275,1732018404438, table=test, region=af2a9e847c9e4d4812af432c436c7ac0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T12:13:28,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=14 2024-11-19T12:13:28,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 19add541d34392b48fdd488f1c19c182, server=351a98347401,36275,1732018404438 in 443 msec 2024-11-19T12:13:28,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=19add541d34392b48fdd488f1c19c182, ASSIGN in 713 msec 2024-11-19T12:13:28,357 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:28,361 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b6d3f4b598ab015d9c62b67f0bac50f, server=351a98347401,36275,1732018404438 because future has completed 2024-11-19T12:13:28,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=28 2024-11-19T12:13:28,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 11d894b29c3ee7cc1e5281c570908571, server=351a98347401,35543,1732018404609 in 453 msec 2024-11-19T12:13:28,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN in 726 msec 2024-11-19T12:13:28,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=26 2024-11-19T12:13:28,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 2b6d3f4b598ab015d9c62b67f0bac50f, server=351a98347401,36275,1732018404438 in 477 msec 2024-11-19T12:13:28,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2b6d3f4b598ab015d9c62b67f0bac50f, ASSIGN in 732 msec 2024-11-19T12:13:28,377 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,45367,1732018404555 2024-11-19T12:13:28,377 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=351a98347401,35543,1732018404609, table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T12:13:28,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef4298fd6621c8e7facf42422bd80be4, server=351a98347401,45367,1732018404555 because future has completed 2024-11-19T12:13:28,381 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=7382ad40044282a358da8130ce023367, regionState=OPEN, openSeqNum=2, regionLocation=351a98347401,35543,1732018404609 2024-11-19T12:13:28,383 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7382ad40044282a358da8130ce023367, server=351a98347401,35543,1732018404609 because future has completed 2024-11-19T12:13:28,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=18 2024-11-19T12:13:28,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure ef4298fd6621c8e7facf42422bd80be4, server=351a98347401,45367,1732018404555 in 529 msec 2024-11-19T12:13:28,388 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ef4298fd6621c8e7facf42422bd80be4, ASSIGN in 746 msec 2024-11-19T12:13:28,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=12 2024-11-19T12:13:28,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 7382ad40044282a358da8130ce023367, server=351a98347401,35543,1732018404609 in 454 msec 2024-11-19T12:13:28,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, ASSIGN in 751 msec 2024-11-19T12:13:28,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=29 2024-11-19T12:13:28,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure af2a9e847c9e4d4812af432c436c7ac0, server=351a98347401,36275,1732018404438 in 497 msec 2024-11-19T12:13:28,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=af2a9e847c9e4d4812af432c436c7ac0, ASSIGN in 756 msec 2024-11-19T12:13:28,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=19 2024-11-19T12:13:28,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure d8bfccd6e991fb7a90ce5bcadc0a3ef5, server=351a98347401,35543,1732018404609 in 451 msec 2024-11-19T12:13:28,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=4 2024-11-19T12:13:28,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN in 762 msec 2024-11-19T12:13:28,407 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T12:13:28,407 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732018408407"}]},"ts":"1732018408407"} 2024-11-19T12:13:28,410 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-19T12:13:28,411 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T12:13:28,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.4390 sec 2024-11-19T12:13:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T12:13:29,139 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-19T12:13:29,139 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-19T12:13:29,141 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T12:13:29,162 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-19T12:13:29,163 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T12:13:29,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-19T12:13:29,164 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T12:13:29,172 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-19T12:13:29,202 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-19T12:13:29,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:13:29,224 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:13:29,226 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:13:29,229 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:13:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b859464d4ffda6f2df23654fb949364f changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f9224ed53fa7acfead92d4143c8626ff changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7382ad40044282a358da8130ce023367 changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ab050043367e9fb30539590d5f965d47 changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5bc8ba883b6bd714e90daec2cb61209e changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d8bfccd6e991fb7a90ce5bcadc0a3ef5 changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 24b360a2b049dbc9c784cdd687321115 changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 11d894b29c3ee7cc1e5281c570908571 changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e257775d8750f12b70161bb0b6387163 changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a78d1de57ec917279b938f869bdbe10d changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 19add541d34392b48fdd488f1c19c182 changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region adc3abaf30d887a0d57a7730313a62b4 changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 963c42e1179165051d86c1560642c124 changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 797a72fe833580fc974f887653d7763a changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2b6d3f4b598ab015d9c62b67f0bac50f changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region af2a9e847c9e4d4812af432c436c7ac0 changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5b9b9ba699dcd89ef4b3398f2beb2455 changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8fbade10b0e1d2be0489df2d6c3fcf9d changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 125c93438c95e8e5f1b8be66c59d7cf9 changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a329354a61adc5325fa166bf564b84eb changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ef4298fd6621c8e7facf42422bd80be4 changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ca6b3c7ec060f344fff683e91e796d33 changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2ec97ffd6b2cc998be397eec7fc0a0bc changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e85bee36dde7095f8d08f860a32613d7 changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4fea5432239dac4bfb1b607057a95e34 changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:13:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-19T12:13:29,287 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-19T12:13:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-19T12:13:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.BalancerClusterState(204): Hosts are {351a98347401=0} racks are {/default-rack=0} 2024-11-19T12:13:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.BalancerClusterState(303): server 0 has 10 regions 2024-11-19T12:13:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.BalancerClusterState(303): server 1 has 8 regions 2024-11-19T12:13:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-19T12:13:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T12:13:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T12:13:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-19T12:13:29,289 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T12:13:29,289 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T12:13:29,289 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-19T12:13:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-19T12:13:29,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-11-19T12:13:29,294 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-11-19T12:13:29,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 475 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-19T12:13:29,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-11-19T12:13:29,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] master.HMaster(2172): balance hri=7382ad40044282a358da8130ce023367, source=351a98347401,35543,1732018404609, destination=351a98347401,36275,1732018404438 2024-11-19T12:13:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:29,771 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:29,773 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=7382ad40044282a358da8130ce023367, regionState=CLOSING, regionLocation=351a98347401,35543,1732018404609 2024-11-19T12:13:29,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE because future has completed 2024-11-19T12:13:29,776 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T12:13:29,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7382ad40044282a358da8130ce023367, server=351a98347401,35543,1732018404609}] 2024-11-19T12:13:29,935 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close 7382ad40044282a358da8130ce023367 2024-11-19T12:13:29,935 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T12:13:29,936 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing 7382ad40044282a358da8130ce023367, disabling compactions & flushes 2024-11-19T12:13:29,936 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,hhh,1732018406961.7382ad40044282a358da8130ce023367. 2024-11-19T12:13:29,936 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732018406961.7382ad40044282a358da8130ce023367. 2024-11-19T12:13:29,936 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732018406961.7382ad40044282a358da8130ce023367. after waiting 0 ms 2024-11-19T12:13:29,936 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732018406961.7382ad40044282a358da8130ce023367. 2024-11-19T12:13:29,942 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/7382ad40044282a358da8130ce023367/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:29,946 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,hhh,1732018406961.7382ad40044282a358da8130ce023367. 2024-11-19T12:13:29,946 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for 7382ad40044282a358da8130ce023367: Waiting for close lock at 1732018409936Running coprocessor pre-close hooks at 1732018409936Disabling compacts and flushes for region at 1732018409936Disabling writes for close at 1732018409936Writing region close event to WAL at 1732018409936Running coprocessor post-close hooks at 1732018409943 (+7 ms)Closed at 1732018409945 (+2 ms) 2024-11-19T12:13:29,946 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding 7382ad40044282a358da8130ce023367 move to 351a98347401,36275,1732018404438 record at close sequenceid=2 2024-11-19T12:13:29,950 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed 7382ad40044282a358da8130ce023367 2024-11-19T12:13:29,951 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=7382ad40044282a358da8130ce023367, regionState=CLOSED 2024-11-19T12:13:29,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7382ad40044282a358da8130ce023367, server=351a98347401,35543,1732018404609 because future has completed 2024-11-19T12:13:29,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-19T12:13:29,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure 7382ad40044282a358da8130ce023367, server=351a98347401,35543,1732018404609 in 178 msec 2024-11-19T12:13:29,958 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE; state=CLOSED, location=351a98347401,36275,1732018404438; forceNewPlan=false, retain=false 2024-11-19T12:13:30,109 INFO [351a98347401:38839 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-19T12:13:30,109 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=7382ad40044282a358da8130ce023367, regionState=OPENING, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:30,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE because future has completed 2024-11-19T12:13:30,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7382ad40044282a358da8130ce023367, server=351a98347401,36275,1732018404438}] 2024-11-19T12:13:30,280 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,hhh,1732018406961.7382ad40044282a358da8130ce023367. 2024-11-19T12:13:30,280 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 7382ad40044282a358da8130ce023367, NAME => 'test,hhh,1732018406961.7382ad40044282a358da8130ce023367.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-19T12:13:30,281 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7382ad40044282a358da8130ce023367 2024-11-19T12:13:30,281 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,hhh,1732018406961.7382ad40044282a358da8130ce023367.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:30,281 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 7382ad40044282a358da8130ce023367 2024-11-19T12:13:30,281 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 7382ad40044282a358da8130ce023367 2024-11-19T12:13:30,283 INFO [StoreOpener-7382ad40044282a358da8130ce023367-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7382ad40044282a358da8130ce023367 2024-11-19T12:13:30,284 INFO [StoreOpener-7382ad40044282a358da8130ce023367-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7382ad40044282a358da8130ce023367 columnFamilyName family 2024-11-19T12:13:30,284 DEBUG [StoreOpener-7382ad40044282a358da8130ce023367-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:30,285 INFO [StoreOpener-7382ad40044282a358da8130ce023367-1 {}] regionserver.HStore(327): Store=7382ad40044282a358da8130ce023367/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:30,285 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 7382ad40044282a358da8130ce023367 2024-11-19T12:13:30,286 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/7382ad40044282a358da8130ce023367 2024-11-19T12:13:30,288 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/7382ad40044282a358da8130ce023367 2024-11-19T12:13:30,288 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 7382ad40044282a358da8130ce023367 2024-11-19T12:13:30,288 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 7382ad40044282a358da8130ce023367 2024-11-19T12:13:30,290 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 7382ad40044282a358da8130ce023367 2024-11-19T12:13:30,291 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 7382ad40044282a358da8130ce023367; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61859024, jitterRate=-0.07822871208190918}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:30,292 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7382ad40044282a358da8130ce023367 2024-11-19T12:13:30,292 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 7382ad40044282a358da8130ce023367: Running coprocessor pre-open hook at 1732018410281Writing region info on filesystem at 1732018410281Initializing all the Stores at 1732018410283 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018410283Cleaning up temporary data from old regions at 1732018410288 (+5 ms)Running coprocessor post-open hooks at 1732018410292 (+4 ms)Region opened successfully at 1732018410292 2024-11-19T12:13:30,293 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732018406961.7382ad40044282a358da8130ce023367., pid=59, masterSystemTime=1732018410269 2024-11-19T12:13:30,295 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732018406961.7382ad40044282a358da8130ce023367. 2024-11-19T12:13:30,295 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,hhh,1732018406961.7382ad40044282a358da8130ce023367. 2024-11-19T12:13:30,296 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=7382ad40044282a358da8130ce023367, regionState=OPEN, openSeqNum=5, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:30,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7382ad40044282a358da8130ce023367, server=351a98347401,36275,1732018404438 because future has completed 2024-11-19T12:13:30,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-11-19T12:13:30,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure 7382ad40044282a358da8130ce023367, server=351a98347401,36275,1732018404438 in 185 msec 2024-11-19T12:13:30,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE in 534 msec 2024-11-19T12:13:30,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-19T12:13:30,375 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T12:13:30,388 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-19T12:13:30,406 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=383, OpenFileDescriptor=642, MaxFileDescriptor=1048576, SystemLoadAverage=314, ProcessCount=11, AvailableMemoryMB=13425 2024-11-19T12:13:30,409 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-19T12:13:30,410 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-19T12:13:30,411 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 351a98347401,35543,1732018404609 2024-11-19T12:13:30,411 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 351a98347401,35543,1732018404609: Simulated kill ***** 2024-11-19T12:13:30,411 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-19T12:13:30,414 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-19T12:13:30,416 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-19T12:13:30,417 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-19T12:13:30,417 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-19T12:13:30,417 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 328107544 }, "NonHeapMemoryUsage": { "committed": 153747456, "init": 7667712, "max": -1, "used": 151492320 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "351a98347401" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "351a98347401" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "351a98347401" } ] } 2024-11-19T12:13:30,420 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38839 {}] master.MasterRpcServices(700): 351a98347401,35543,1732018404609 reported a fatal error: ***** ABORTING region server 351a98347401,35543,1732018404609: Simulated kill ***** 2024-11-19T12:13:30,423 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '351a98347401,35543,1732018404609' ***** 2024-11-19T12:13:30,423 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-19T12:13:30,423 INFO [RS:2;351a98347401:35543 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T12:13:30,423 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T12:13:30,423 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T12:13:30,423 INFO [RS:2;351a98347401:35543 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-19T12:13:30,424 INFO [RS:2;351a98347401:35543 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-19T12:13:30,424 INFO [RS:2;351a98347401:35543 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T12:13:30,424 INFO [RS:2;351a98347401:35543 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;351a98347401:35543. 2024-11-19T12:13:30,424 DEBUG [RS:2;351a98347401:35543 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:13:30,429 DEBUG [RS:2;351a98347401:35543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:13:30,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741836_1012 (size=33749) 2024-11-19T12:13:30,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741836_1012 (size=33749) 2024-11-19T12:13:30,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741836_1012 (size=33749) 2024-11-19T12:13:30,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741833_1009 (size=2769) 2024-11-19T12:13:30,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741833_1009 (size=2769) 2024-11-19T12:13:30,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741833_1009 (size=2769) 2024-11-19T12:13:30,441 DEBUG [RS:2;351a98347401:35543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:13:30,441 INFO [RS:2;351a98347401:35543 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T12:13:30,442 INFO [RS:2;351a98347401:35543 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35543 2024-11-19T12:13:30,461 INFO [regionserver/351a98347401:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T12:13:30,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T12:13:30,511 INFO [RS:2;351a98347401:35543 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T12:13:30,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35543-0x1015303718b0003, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/351a98347401,35543,1732018404609 2024-11-19T12:13:30,521 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [351a98347401,35543,1732018404609] 2024-11-19T12:13:30,531 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/351a98347401,35543,1732018404609 already deleted, retry=false 2024-11-19T12:13:30,532 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 351a98347401,35543,1732018404609 on 351a98347401,38839,1732018403620 2024-11-19T12:13:30,536 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:30,540 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:30,540 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=60 for 351a98347401,35543,1732018404609 (carryingMeta=true) 351a98347401,35543,1732018404609/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1810b6b[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-19T12:13:30,541 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true, isMeta: true 2024-11-19T12:13:30,543 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting 2024-11-19T12:13:30,544 INFO [PEWorker-1 {}] master.SplitWALManager(105): 351a98347401,35543,1732018404609 WAL count=1, meta=true 2024-11-19T12:13:30,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta}] 2024-11-19T12:13:30,552 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=351a98347401,36275,1732018404438 2024-11-19T12:13:30,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438}] 2024-11-19T12:13:30,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35543-0x1015303718b0003, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T12:13:30,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35543-0x1015303718b0003, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T12:13:30,622 INFO [RS:2;351a98347401:35543 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T12:13:30,622 INFO [RS:2;351a98347401:35543 {}] regionserver.HRegionServer(1031): Exiting; stopping=351a98347401,35543,1732018404609; zookeeper connection closed. 2024-11-19T12:13:30,623 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5df4b7c9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5df4b7c9 2024-11-19T12:13:30,625 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T12:13:30,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36275 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-11-19T12:13:30,743 INFO [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, size=33.0 K (33749bytes) 2024-11-19T12:13:30,743 DEBUG [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-19T12:13:30,744 DEBUG [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-19T12:13:30,744 INFO [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta 2024-11-19T12:13:30,749 INFO [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta after 3ms 2024-11-19T12:13:30,756 INFO [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta took 15ms 2024-11-19T12:13:30,765 DEBUG [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-19T12:13:30,770 INFO [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta.temp 2024-11-19T12:13:30,774 INFO [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta.temp 2024-11-19T12:13:30,777 DEBUG [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta so closing down 2024-11-19T12:13:30,778 DEBUG [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-19T12:13:30,778 INFO [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-11-19T12:13:30,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741864_1040 (size=33508) 2024-11-19T12:13:30,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741864_1040 (size=33508) 2024-11-19T12:13:30,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741864_1040 (size=33508) 2024-11-19T12:13:30,789 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta.temp (wrote 61 edits, skipped 0 edits in 3 ms) 2024-11-19T12:13:30,792 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta.temp to hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-19T12:13:30,797 INFO [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 35 ms; skipped=1; WAL=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-11-19T12:13:30,797 DEBUG [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, journal: Splitting hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, size=33.0 K (33749bytes) at 1732018410743Creating recovered edits writer path=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta.temp at 1732018410774 (+31 ms)Finishing writing output for hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta so closing down at 1732018410778 (+4 ms)3 split writer threads finished at 1732018410778Closed recovered edits writer path=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta.temp (wrote 61 edits, skipped 0 edits in 3 ms) at 1732018410789 (+11 ms)Rename recovered edits hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta.temp to hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1732018410792 (+3 ms)Processed 62 edits across 1 Regions in 35 ms; skipped=1; WAL=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1732018410797 (+5 ms) 2024-11-19T12:13:30,797 DEBUG [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta 2024-11-19T12:13:30,799 DEBUG [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-19T12:13:30,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-11-19T12:13:30,808 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta to hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/oldWALs 2024-11-19T12:13:30,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-19T12:13:30,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 in 254 msec 2024-11-19T12:13:30,813 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=351a98347401,36275,1732018404438 2024-11-19T12:13:30,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-19T12:13:30,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 in 267 msec 2024-11-19T12:13:30,819 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting dir is empty, no logs to split. 2024-11-19T12:13:30,819 INFO [PEWorker-1 {}] master.SplitWALManager(105): 351a98347401,35543,1732018404609 WAL count=0, meta=true 2024-11-19T12:13:30,819 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 351a98347401,35543,1732018404609 WAL splitting is done? wals=0, meta=true 2024-11-19T12:13:30,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T12:13:30,823 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:30,824 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T12:13:30,967 INFO [regionserver/351a98347401:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-19T12:13:30,967 INFO [regionserver/351a98347401:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-19T12:13:30,974 DEBUG [351a98347401:38839 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-19T12:13:30,975 DEBUG [351a98347401:38839 {}] balancer.BalancerClusterState(204): Hosts are {351a98347401=0} racks are {/default-rack=0} 2024-11-19T12:13:30,975 DEBUG [351a98347401:38839 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T12:13:30,975 DEBUG [351a98347401:38839 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T12:13:30,975 DEBUG [351a98347401:38839 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T12:13:30,975 DEBUG [351a98347401:38839 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T12:13:30,975 INFO [351a98347401:38839 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T12:13:30,975 INFO [351a98347401:38839 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T12:13:30,975 DEBUG [351a98347401:38839 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T12:13:30,976 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:30,977 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPENING 2024-11-19T12:13:31,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:31,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:31,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:31,048 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:31,048 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:31,048 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:31,048 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:31,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438}] 2024-11-19T12:13:31,206 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T12:13:31,206 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T12:13:31,206 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T12:13:31,209 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=351a98347401%2C36275%2C1732018404438.meta, suffix=.meta, logDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,36275,1732018404438, archiveDir=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/oldWALs, maxLogs=32 2024-11-19T12:13:31,224 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,36275,1732018404438/351a98347401%2C36275%2C1732018404438.meta.1732018411212.meta, exclude list is [], retry=0 2024-11-19T12:13:31,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40579,DS-fc24d53f-eb0d-4d9b-b2d4-feb49c03511f,DISK] 2024-11-19T12:13:31,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46857,DS-73e25d91-6f88-4582-919b-1060a325c9ba,DISK] 2024-11-19T12:13:31,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46231,DS-598e6c09-e04f-4361-a363-9b507e9888f8,DISK] 2024-11-19T12:13:31,229 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,36275,1732018404438/351a98347401%2C36275%2C1732018404438.meta.1732018411212.meta 2024-11-19T12:13:31,231 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45779:45779),(127.0.0.1/127.0.0.1:40699:40699),(127.0.0.1/127.0.0.1:39493:39493)] 2024-11-19T12:13:31,231 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:13:31,231 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T12:13:31,231 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T12:13:31,232 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T12:13:31,232 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T12:13:31,232 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:31,232 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T12:13:31,232 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T12:13:31,234 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T12:13:31,236 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T12:13:31,236 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:31,237 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:13:31,237 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T12:13:31,238 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T12:13:31,238 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:31,239 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:13:31,239 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T12:13:31,240 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T12:13:31,241 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:31,241 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:13:31,241 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T12:13:31,243 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T12:13:31,243 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:31,243 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:13:31,243 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T12:13:31,244 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740 2024-11-19T12:13:31,247 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740 2024-11-19T12:13:31,248 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-19T12:13:31,261 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-19T12:13:31,264 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-11-19T12:13:31,327 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/.tmp/info/702b681a5fe3407cbc9b94784c250899 is 138, key is test,bbb,1732018406961.e257775d8750f12b70161bb0b6387163./info:regioninfo/1732018408281/Put/seqid=0 2024-11-19T12:13:31,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741866_1042 (size=36385) 2024-11-19T12:13:31,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741866_1042 (size=36385) 2024-11-19T12:13:31,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741866_1042 (size=36385) 2024-11-19T12:13:31,344 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/.tmp/info/702b681a5fe3407cbc9b94784c250899 2024-11-19T12:13:31,411 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/.tmp/ns/e3ec5b18de0141249d091fdf9c4da858 is 43, key is default/ns:d/1732018406737/Put/seqid=0 2024-11-19T12:13:31,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741867_1043 (size=5153) 2024-11-19T12:13:31,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741867_1043 (size=5153) 2024-11-19T12:13:31,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741867_1043 (size=5153) 2024-11-19T12:13:31,426 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/.tmp/ns/e3ec5b18de0141249d091fdf9c4da858 2024-11-19T12:13:31,454 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/.tmp/table/fad279df0ad74cf984d03276660e75ad is 40, key is test/table:state/1732018408407/Put/seqid=0 2024-11-19T12:13:31,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741868_1044 (size=5165) 2024-11-19T12:13:31,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741868_1044 (size=5165) 2024-11-19T12:13:31,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741868_1044 (size=5165) 2024-11-19T12:13:31,465 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/.tmp/table/fad279df0ad74cf984d03276660e75ad 2024-11-19T12:13:31,477 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/.tmp/info/702b681a5fe3407cbc9b94784c250899 as hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/info/702b681a5fe3407cbc9b94784c250899 2024-11-19T12:13:31,487 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/info/702b681a5fe3407cbc9b94784c250899, entries=273, sequenceid=64, filesize=35.5 K 2024-11-19T12:13:31,490 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/.tmp/ns/e3ec5b18de0141249d091fdf9c4da858 as hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/ns/e3ec5b18de0141249d091fdf9c4da858 2024-11-19T12:13:31,498 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/ns/e3ec5b18de0141249d091fdf9c4da858, entries=2, sequenceid=64, filesize=5.0 K 2024-11-19T12:13:31,500 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/.tmp/table/fad279df0ad74cf984d03276660e75ad as hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/table/fad279df0ad74cf984d03276660e75ad 2024-11-19T12:13:31,511 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/table/fad279df0ad74cf984d03276660e75ad, entries=2, sequenceid=64, filesize=5.0 K 2024-11-19T12:13:31,517 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 250ms, sequenceid=64, compaction requested=false; wal=null 2024-11-19T12:13:31,519 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-19T12:13:31,522 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-19T12:13:31,524 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T12:13:31,524 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T12:13:31,525 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T12:13:31,527 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T12:13:31,530 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-11-19T12:13:31,532 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65167774, jitterRate=-0.028924494981765747}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T12:13:31,532 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T12:13:31,533 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732018411232Writing region info on filesystem at 1732018411232Initializing all the Stores at 1732018411234 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732018411234Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732018411234Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018411234Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732018411234Obtaining lock to block concurrent updates at 1732018411264 (+30 ms)Preparing flush snapshotting stores in 1588230740 at 1732018411264Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1732018411267 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732018411267Flushing 1588230740/info: creating writer at 1732018411268 (+1 ms)Flushing 1588230740/info: appending metadata at 1732018411317 (+49 ms)Flushing 1588230740/info: closing flushed file at 1732018411319 (+2 ms)Flushing 1588230740/ns: creating writer at 1732018411395 (+76 ms)Flushing 1588230740/ns: appending metadata at 1732018411410 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732018411410Flushing 1588230740/table: creating writer at 1732018411437 (+27 ms)Flushing 1588230740/table: appending metadata at 1732018411453 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732018411453Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5458245a: reopening flushed file at 1732018411475 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21b3a21d: reopening flushed file at 1732018411488 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31dc777f: reopening flushed file at 1732018411499 (+11 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 250ms, sequenceid=64, compaction requested=false; wal=null at 1732018411517 (+18 ms)Cleaning up temporary data from old regions at 1732018411524 (+7 ms)Running coprocessor post-open hooks at 1732018411532 (+8 ms)Region opened successfully at 1732018411532 2024-11-19T12:13:31,534 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1732018411200 2024-11-19T12:13:31,537 DEBUG [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T12:13:31,537 INFO [RS_OPEN_META-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T12:13:31,538 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:31,539 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:31,563 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T12:13:31,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:31,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:31,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:31,595 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 2024-11-19T12:13:31,595 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:31,595 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:31,595 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:31,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-19T12:13:31,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 in 547 msec 2024-11-19T12:13:31,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-11-19T12:13:31,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 777 msec 2024-11-19T12:13:31,602 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 351a98347401,35543,1732018404609 had 9 regions 2024-11-19T12:13:31,604 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true, isMeta: false 2024-11-19T12:13:31,606 INFO [PEWorker-1 {}] master.SplitWALManager(105): 351a98347401,35543,1732018404609 WAL count=1, meta=false 2024-11-19T12:13:31,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120}] 2024-11-19T12:13:31,608 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=351a98347401,45367,1732018404555 2024-11-19T12:13:31,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555}] 2024-11-19T12:13:31,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45367 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-11-19T12:13:31,783 INFO [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.1732018406120, size=2.7 K (2769bytes) 2024-11-19T12:13:31,783 INFO [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.1732018406120 2024-11-19T12:13:31,784 INFO [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.1732018406120 after 1ms 2024-11-19T12:13:31,787 INFO [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.1732018406120 took 4ms 2024-11-19T12:13:31,790 DEBUG [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for d8bfccd6e991fb7a90ce5bcadc0a3ef5: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-19T12:13:31,792 DEBUG [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 7382ad40044282a358da8130ce023367: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-19T12:13:31,794 DEBUG [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 11d894b29c3ee7cc1e5281c570908571: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-19T12:13:31,795 DEBUG [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for c48a9e94bd9fcbd6f8c1f6ae2b6c80d3: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-19T12:13:31,797 DEBUG [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for b859464d4ffda6f2df23654fb949364f: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-19T12:13:31,798 DEBUG [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for ab050043367e9fb30539590d5f965d47: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-19T12:13:31,800 DEBUG [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for f9224ed53fa7acfead92d4143c8626ff: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-19T12:13:31,802 DEBUG [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 5bc8ba883b6bd714e90daec2cb61209e: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-19T12:13:31,804 DEBUG [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 24b360a2b049dbc9c784cdd687321115: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-19T12:13:31,804 DEBUG [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.1732018406120 so closing down 2024-11-19T12:13:31,804 DEBUG [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-19T12:13:31,804 INFO [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-11-19T12:13:31,804 INFO [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 17 ms; skipped=10; WAL=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.1732018406120, size=2.7 K, length=2769, corrupted=false, cancelled=false 2024-11-19T12:13:31,804 DEBUG [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.1732018406120, journal: Splitting hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.1732018406120, size=2.7 K (2769bytes) at 1732018411783Finishing writing output for hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.1732018406120 so closing down at 1732018411804 (+21 ms)3 split writer threads finished at 1732018411804Processed 10 edits across 0 Regions in 17 ms; skipped=10; WAL=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.1732018406120, size=2.7 K, length=2769, corrupted=false, cancelled=false at 1732018411804 2024-11-19T12:13:31,805 DEBUG [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.1732018406120 2024-11-19T12:13:31,805 DEBUG [RS_LOG_REPLAY_OPS-regionserver/351a98347401:0-0 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-19T12:13:31,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38839 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-11-19T12:13:31,809 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting/351a98347401%2C35543%2C1732018404609.1732018406120 to hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/oldWALs 2024-11-19T12:13:31,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-19T12:13:31,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 in 200 msec 2024-11-19T12:13:31,815 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=351a98347401,45367,1732018404555 2024-11-19T12:13:31,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-11-19T12:13:31,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 in 209 msec 2024-11-19T12:13:31,821 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/WALs/351a98347401,35543,1732018404609-splitting dir is empty, no logs to split. 2024-11-19T12:13:31,821 INFO [PEWorker-1 {}] master.SplitWALManager(105): 351a98347401,35543,1732018404609 WAL count=0, meta=false 2024-11-19T12:13:31,821 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 351a98347401,35543,1732018404609 WAL splitting is done? wals=0, meta=false 2024-11-19T12:13:31,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN}] 2024-11-19T12:13:31,825 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN 2024-11-19T12:13:31,826 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T12:13:31,827 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:31,828 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T12:13:31,829 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:31,829 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T12:13:31,830 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:31,831 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T12:13:31,831 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:31,832 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T12:13:31,833 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:31,834 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T12:13:31,835 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:31,836 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T12:13:31,837 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:31,838 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T12:13:31,977 DEBUG [351a98347401:38839 {}] balancer.BalancerClusterState(204): Hosts are {351a98347401=0} racks are {/default-rack=0} 2024-11-19T12:13:31,977 DEBUG [351a98347401:38839 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T12:13:31,977 DEBUG [351a98347401:38839 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T12:13:31,977 DEBUG [351a98347401:38839 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T12:13:31,977 DEBUG [351a98347401:38839 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T12:13:31,977 INFO [351a98347401:38839 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T12:13:31,977 INFO [351a98347401:38839 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T12:13:31,977 DEBUG [351a98347401:38839 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T12:13:31,978 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:31,979 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:31,979 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=ab050043367e9fb30539590d5f965d47, regionState=OPENING, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:31,980 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, regionLocation=351a98347401,36275,1732018404438 2024-11-19T12:13:31,980 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 351a98347401:35543 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 351a98347401/172.17.0.2:35543 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:13:31,981 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, regionLocation=351a98347401,45367,1732018404555 2024-11-19T12:13:31,981 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, regionLocation=351a98347401,45367,1732018404555 2024-11-19T12:13:31,982 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, regionLocation=351a98347401,45367,1732018404555 2024-11-19T12:13:31,983 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, regionLocation=351a98347401,45367,1732018404555 2024-11-19T12:13:31,983 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=351a98347401,35543,1732018404609, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=351a98347401,35543,1732018404609, seqNum=-1, error=java.net.ConnectException: Call to address=351a98347401:35543 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 351a98347401/172.17.0.2:35543 2024-11-19T12:13:31,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=351a98347401,35543,1732018404609, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-19T12:13:31,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=351a98347401,35543,1732018404609, seqNum=-1 from cache 2024-11-19T12:13:31,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 351a98347401:35543 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 351a98347401/172.17.0.2:35543 2024-11-19T12:13:31,986 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 351a98347401:35543 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 351a98347401/172.17.0.2:35543 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:13:31,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=351a98347401,35543,1732018404609, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=351a98347401:35543 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 351a98347401/172.17.0.2:35543 2024-11-19T12:13:31,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=351a98347401,35543,1732018404609, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=351a98347401:35543 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 351a98347401/172.17.0.2:35543 2024-11-19T12:13:31,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=351a98347401,35543,1732018404609, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=351a98347401:35543 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 351a98347401/172.17.0.2:35543 2024-11-19T12:13:31,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=351a98347401,35543,1732018404609, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=351a98347401:35543 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 351a98347401/172.17.0.2:35543 2024-11-19T12:13:31,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=351a98347401,35543,1732018404609, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=351a98347401:35543 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 351a98347401/172.17.0.2:35543 2024-11-19T12:13:31,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=351a98347401,35543,1732018404609, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=351a98347401:35543 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 351a98347401/172.17.0.2:35543 2024-11-19T12:13:31,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=351a98347401,35543,1732018404609, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=351a98347401:35543 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 351a98347401/172.17.0.2:35543 2024-11-19T12:13:31,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 351a98347401:35543 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 351a98347401/172.17.0.2:35543 2024-11-19T12:13:32,088 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:13:32,088 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=351a98347401,36275,1732018404438, seqNum=-1] 2024-11-19T12:13:32,088 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:13:32,090 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57735, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:13:32,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN because future has completed 2024-11-19T12:13:32,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, server=351a98347401,36275,1732018404438}] 2024-11-19T12:13:32,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN because future has completed 2024-11-19T12:13:32,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN because future has completed 2024-11-19T12:13:32,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN because future has completed 2024-11-19T12:13:32,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8bfccd6e991fb7a90ce5bcadc0a3ef5, server=351a98347401,36275,1732018404438}] 2024-11-19T12:13:32,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN because future has completed 2024-11-19T12:13:32,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN because future has completed 2024-11-19T12:13:32,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN because future has completed 2024-11-19T12:13:32,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11d894b29c3ee7cc1e5281c570908571, server=351a98347401,45367,1732018404555}] 2024-11-19T12:13:32,102 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN because future has completed 2024-11-19T12:13:32,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24b360a2b049dbc9c784cdd687321115, server=351a98347401,45367,1732018404555}] 2024-11-19T12:13:32,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438}] 2024-11-19T12:13:32,116 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T12:13:32,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab050043367e9fb30539590d5f965d47, server=351a98347401,36275,1732018404438}] 2024-11-19T12:13:32,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9224ed53fa7acfead92d4143c8626ff, server=351a98347401,45367,1732018404555}] 2024-11-19T12:13:32,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bc8ba883b6bd714e90daec2cb61209e, server=351a98347401,45367,1732018404555}] 2024-11-19T12:13:32,168 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-19T12:13:32,253 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,,1732018406961.b859464d4ffda6f2df23654fb949364f. 2024-11-19T12:13:32,254 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => b859464d4ffda6f2df23654fb949364f, NAME => 'test,,1732018406961.b859464d4ffda6f2df23654fb949364f.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-19T12:13:32,254 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:32,254 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,,1732018406961.b859464d4ffda6f2df23654fb949364f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:32,255 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:32,255 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:32,257 INFO [StoreOpener-b859464d4ffda6f2df23654fb949364f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:32,259 INFO [StoreOpener-b859464d4ffda6f2df23654fb949364f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b859464d4ffda6f2df23654fb949364f columnFamilyName family 2024-11-19T12:13:32,259 DEBUG [StoreOpener-b859464d4ffda6f2df23654fb949364f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:32,260 INFO [StoreOpener-b859464d4ffda6f2df23654fb949364f-1 {}] regionserver.HStore(327): Store=b859464d4ffda6f2df23654fb949364f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:32,260 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:32,261 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:32,261 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115. 2024-11-19T12:13:32,262 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 24b360a2b049dbc9c784cdd687321115, NAME => 'test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-19T12:13:32,262 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:32,262 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:32,262 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:32,262 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:32,263 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:32,264 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:32,264 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:32,264 INFO [StoreOpener-24b360a2b049dbc9c784cdd687321115-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:32,265 INFO [StoreOpener-24b360a2b049dbc9c784cdd687321115-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24b360a2b049dbc9c784cdd687321115 columnFamilyName family 2024-11-19T12:13:32,266 DEBUG [StoreOpener-24b360a2b049dbc9c784cdd687321115-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:32,266 INFO [StoreOpener-24b360a2b049dbc9c784cdd687321115-1 {}] regionserver.HStore(327): Store=24b360a2b049dbc9c784cdd687321115/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:32,266 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:32,266 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:32,268 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:32,268 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened b859464d4ffda6f2df23654fb949364f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63695715, jitterRate=-0.050859883427619934}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:32,268 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:32,268 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for b859464d4ffda6f2df23654fb949364f: Running coprocessor pre-open hook at 1732018412255Writing region info on filesystem at 1732018412255Initializing all the Stores at 1732018412257 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018412257Cleaning up temporary data from old regions at 1732018412264 (+7 ms)Running coprocessor post-open hooks at 1732018412268 (+4 ms)Region opened successfully at 1732018412268 2024-11-19T12:13:32,269 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:32,270 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:32,270 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:32,272 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:32,272 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732018406961.b859464d4ffda6f2df23654fb949364f., pid=79, masterSystemTime=1732018412246 2024-11-19T12:13:32,273 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 24b360a2b049dbc9c784cdd687321115; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72681205, jitterRate=0.08303435146808624}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:32,273 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:32,273 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 24b360a2b049dbc9c784cdd687321115: Running coprocessor pre-open hook at 1732018412263Writing region info on filesystem at 1732018412263Initializing all the Stores at 1732018412264 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018412264Cleaning up temporary data from old regions at 1732018412270 (+6 ms)Running coprocessor post-open hooks at 1732018412273 (+3 ms)Region opened successfully at 1732018412273 2024-11-19T12:13:32,274 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115., pid=78, masterSystemTime=1732018412255 2024-11-19T12:13:32,275 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732018406961.b859464d4ffda6f2df23654fb949364f. 2024-11-19T12:13:32,275 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,,1732018406961.b859464d4ffda6f2df23654fb949364f. 2024-11-19T12:13:32,275 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3. 2024-11-19T12:13:32,275 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, NAME => 'test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-19T12:13:32,276 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:32,276 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:32,276 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:32,276 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:32,277 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115. 2024-11-19T12:13:32,278 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115. 2024-11-19T12:13:32,278 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff. 2024-11-19T12:13:32,278 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => f9224ed53fa7acfead92d4143c8626ff, NAME => 'test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-19T12:13:32,279 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:32,279 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-19T12:13:32,279 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:32,279 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,279 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:32,279 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:32,280 INFO [StoreOpener-c48a9e94bd9fcbd6f8c1f6ae2b6c80d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:32,279 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,284 INFO [StoreOpener-c48a9e94bd9fcbd6f8c1f6ae2b6c80d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 columnFamilyName family 2024-11-19T12:13:32,285 DEBUG [StoreOpener-c48a9e94bd9fcbd6f8c1f6ae2b6c80d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:32,285 INFO [StoreOpener-f9224ed53fa7acfead92d4143c8626ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:32,285 INFO [StoreOpener-c48a9e94bd9fcbd6f8c1f6ae2b6c80d3-1 {}] regionserver.HStore(327): Store=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:32,286 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:32,286 INFO [StoreOpener-f9224ed53fa7acfead92d4143c8626ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9224ed53fa7acfead92d4143c8626ff columnFamilyName family 2024-11-19T12:13:32,287 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:32,287 DEBUG [StoreOpener-f9224ed53fa7acfead92d4143c8626ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:32,287 INFO [StoreOpener-f9224ed53fa7acfead92d4143c8626ff-1 {}] regionserver.HStore(327): Store=f9224ed53fa7acfead92d4143c8626ff/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:32,288 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:32,288 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:32,289 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:32,289 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:32,289 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:32,288 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-19T12:13:32,290 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:32,290 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bc8ba883b6bd714e90daec2cb61209e, server=351a98347401,45367,1732018404555 2024-11-19T12:13:32,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,290 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:32,291 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:32,291 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:32,291 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bc8ba883b6bd714e90daec2cb61209e, server=351a98347401,45367,1732018404555 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,291 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:32,293 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened c48a9e94bd9fcbd6f8c1f6ae2b6c80d3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64041676, jitterRate=-0.045704662799835205}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:32,293 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:32,293 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:32,293 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for c48a9e94bd9fcbd6f8c1f6ae2b6c80d3: Running coprocessor pre-open hook at 1732018412276Writing region info on filesystem at 1732018412276Initializing all the Stores at 1732018412277 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018412278 (+1 ms)Cleaning up temporary data from old regions at 1732018412289 (+11 ms)Running coprocessor post-open hooks at 1732018412293 (+4 ms)Region opened successfully at 1732018412293 2024-11-19T12:13:32,294 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened f9224ed53fa7acfead92d4143c8626ff; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70962062, jitterRate=0.0574171245098114}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:32,294 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3., pid=75, masterSystemTime=1732018412246 2024-11-19T12:13:32,294 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:32,294 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for f9224ed53fa7acfead92d4143c8626ff: Running coprocessor pre-open hook at 1732018412279Writing region info on filesystem at 1732018412279Initializing all the Stores at 1732018412284 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018412284Cleaning up temporary data from old regions at 1732018412291 (+7 ms)Running coprocessor post-open hooks at 1732018412294 (+3 ms)Region opened successfully at 1732018412294 2024-11-19T12:13:32,295 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff., pid=81, masterSystemTime=1732018412255 2024-11-19T12:13:32,296 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3. 2024-11-19T12:13:32,297 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3. 2024-11-19T12:13:32,297 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,iii,1732018406961.ab050043367e9fb30539590d5f965d47. 2024-11-19T12:13:32,297 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => ab050043367e9fb30539590d5f965d47, NAME => 'test,iii,1732018406961.ab050043367e9fb30539590d5f965d47.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-19T12:13:32,297 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:32,297 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,iii,1732018406961.ab050043367e9fb30539590d5f965d47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:32,298 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:32,298 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:32,299 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff. 2024-11-19T12:13:32,299 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff. 2024-11-19T12:13:32,299 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e. 2024-11-19T12:13:32,299 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => 5bc8ba883b6bd714e90daec2cb61209e, NAME => 'test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-19T12:13:32,299 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:32,299 INFO [StoreOpener-ab050043367e9fb30539590d5f965d47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:32,300 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:32,300 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for 5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:32,300 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for 5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:32,301 INFO [StoreOpener-ab050043367e9fb30539590d5f965d47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab050043367e9fb30539590d5f965d47 columnFamilyName family 2024-11-19T12:13:32,301 DEBUG [StoreOpener-ab050043367e9fb30539590d5f965d47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:32,301 INFO [StoreOpener-5bc8ba883b6bd714e90daec2cb61209e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:32,302 INFO [StoreOpener-ab050043367e9fb30539590d5f965d47-1 {}] regionserver.HStore(327): Store=ab050043367e9fb30539590d5f965d47/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:32,302 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:32,302 INFO [StoreOpener-5bc8ba883b6bd714e90daec2cb61209e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bc8ba883b6bd714e90daec2cb61209e columnFamilyName family 2024-11-19T12:13:32,302 DEBUG [StoreOpener-5bc8ba883b6bd714e90daec2cb61209e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:32,303 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:32,303 INFO [StoreOpener-5bc8ba883b6bd714e90daec2cb61209e-1 {}] regionserver.HStore(327): Store=5bc8ba883b6bd714e90daec2cb61209e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:32,303 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for 5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:32,304 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:32,304 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:32,304 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:32,305 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:32,305 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:32,305 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for 5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:32,305 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for 5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:32,306 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:32,308 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened ab050043367e9fb30539590d5f965d47; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61848226, jitterRate=-0.07838961482048035}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:32,309 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for 5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:32,309 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:32,309 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for ab050043367e9fb30539590d5f965d47: Running coprocessor pre-open hook at 1732018412298Writing region info on filesystem at 1732018412298Initializing all the Stores at 1732018412299 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018412299Cleaning up temporary data from old regions at 1732018412305 (+6 ms)Running coprocessor post-open hooks at 1732018412309 (+4 ms)Region opened successfully at 1732018412309 2024-11-19T12:13:32,310 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732018406961.ab050043367e9fb30539590d5f965d47., pid=80, masterSystemTime=1732018412246 2024-11-19T12:13:32,310 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened 5bc8ba883b6bd714e90daec2cb61209e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66888554, jitterRate=-0.003282874822616577}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:32,310 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:32,310 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for 5bc8ba883b6bd714e90daec2cb61209e: Running coprocessor pre-open hook at 1732018412300Writing region info on filesystem at 1732018412300Initializing all the Stores at 1732018412301 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018412301Cleaning up temporary data from old regions at 1732018412305 (+4 ms)Running coprocessor post-open hooks at 1732018412310 (+5 ms)Region opened successfully at 1732018412310 2024-11-19T12:13:32,311 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e., pid=82, masterSystemTime=1732018412255 2024-11-19T12:13:32,312 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732018406961.ab050043367e9fb30539590d5f965d47. 2024-11-19T12:13:32,312 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened test,iii,1732018406961.ab050043367e9fb30539590d5f965d47. 2024-11-19T12:13:32,313 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5. 2024-11-19T12:13:32,313 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => d8bfccd6e991fb7a90ce5bcadc0a3ef5, NAME => 'test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-19T12:13:32,313 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:32,313 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:32,313 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:32,313 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:32,314 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e. 2024-11-19T12:13:32,314 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(153): Opened test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e. 2024-11-19T12:13:32,314 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571. 2024-11-19T12:13:32,314 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 11d894b29c3ee7cc1e5281c570908571, NAME => 'test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-19T12:13:32,315 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:32,315 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:13:32,315 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:32,315 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:32,317 INFO [StoreOpener-d8bfccd6e991fb7a90ce5bcadc0a3ef5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:32,317 INFO [StoreOpener-11d894b29c3ee7cc1e5281c570908571-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:32,318 INFO [StoreOpener-d8bfccd6e991fb7a90ce5bcadc0a3ef5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d8bfccd6e991fb7a90ce5bcadc0a3ef5 columnFamilyName family 2024-11-19T12:13:32,318 DEBUG [StoreOpener-d8bfccd6e991fb7a90ce5bcadc0a3ef5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:32,319 INFO [StoreOpener-11d894b29c3ee7cc1e5281c570908571-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11d894b29c3ee7cc1e5281c570908571 columnFamilyName family 2024-11-19T12:13:32,319 DEBUG [StoreOpener-11d894b29c3ee7cc1e5281c570908571-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:13:32,319 INFO [StoreOpener-d8bfccd6e991fb7a90ce5bcadc0a3ef5-1 {}] regionserver.HStore(327): Store=d8bfccd6e991fb7a90ce5bcadc0a3ef5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:32,319 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:32,319 INFO [StoreOpener-11d894b29c3ee7cc1e5281c570908571-1 {}] regionserver.HStore(327): Store=11d894b29c3ee7cc1e5281c570908571/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:13:32,319 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:32,320 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:32,320 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:32,321 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:32,321 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:32,321 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:32,321 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:32,321 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:32,321 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:32,323 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:32,323 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:32,324 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 11d894b29c3ee7cc1e5281c570908571; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60940045, jitterRate=-0.0919225662946701}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:32,324 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:32,324 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened d8bfccd6e991fb7a90ce5bcadc0a3ef5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69418942, jitterRate=0.034422844648361206}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:13:32,324 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:32,324 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 11d894b29c3ee7cc1e5281c570908571: Running coprocessor pre-open hook at 1732018412315Writing region info on filesystem at 1732018412315Initializing all the Stores at 1732018412317 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018412317Cleaning up temporary data from old regions at 1732018412321 (+4 ms)Running coprocessor post-open hooks at 1732018412324 (+3 ms)Region opened successfully at 1732018412324 2024-11-19T12:13:32,324 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for d8bfccd6e991fb7a90ce5bcadc0a3ef5: Running coprocessor pre-open hook at 1732018412313Writing region info on filesystem at 1732018412313Initializing all the Stores at 1732018412315 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732018412315Cleaning up temporary data from old regions at 1732018412321 (+6 ms)Running coprocessor post-open hooks at 1732018412324 (+3 ms)Region opened successfully at 1732018412324 2024-11-19T12:13:32,326 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571., pid=77, masterSystemTime=1732018412255 2024-11-19T12:13:32,326 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5., pid=76, masterSystemTime=1732018412246 2024-11-19T12:13:32,328 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571. 2024-11-19T12:13:32,328 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571. 2024-11-19T12:13:32,329 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:32,329 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,329 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:32,329 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:32,330 DEBUG [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5. 2024-11-19T12:13:32,330 INFO [RS_OPEN_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5. 2024-11-19T12:13:32,330 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:32,330 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:32,330 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:32,331 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:32,331 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:32,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:32,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,341 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:32,342 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:32,342 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:32,348 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:32,348 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,348 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,348 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:32,348 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,348 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,348 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,348 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,348 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,348 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8bfccd6e991fb7a90ce5bcadc0a3ef5, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11d894b29c3ee7cc1e5281c570908571, server=351a98347401,45367,1732018404555 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24b360a2b049dbc9c784cdd687321115, server=351a98347401,45367,1732018404555 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab050043367e9fb30539590d5f965d47, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9224ed53fa7acfead92d4143c8626ff, server=351a98347401,45367,1732018404555 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bc8ba883b6bd714e90daec2cb61209e, server=351a98347401,45367,1732018404555 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,349 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,349 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,349 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,349 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,350 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,350 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,350 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,350 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,350 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,350 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,350 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:32,350 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,350 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN 2024-11-19T12:13:32,350 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,350 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8bfccd6e991fb7a90ce5bcadc0a3ef5, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,350 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11d894b29c3ee7cc1e5281c570908571, server=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,350 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24b360a2b049dbc9c784cdd687321115, server=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,350 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,350 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab050043367e9fb30539590d5f965d47, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,350 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9224ed53fa7acfead92d4143c8626ff, server=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,350 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bc8ba883b6bd714e90daec2cb61209e, server=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,351 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 19 msec 2024-11-19T12:13:32,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:32,353 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:32,353 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:32,399 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,406 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,406 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,406 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,407 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,408 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN to state=OFFLINE, location=null, table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 to restore RIT 2024-11-19T12:13:32,408 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN to state=OFFLINE, location=null, table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5 to restore RIT 2024-11-19T12:13:32,408 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN to state=OFFLINE, location=null, table=test, region=b859464d4ffda6f2df23654fb949364f to restore RIT 2024-11-19T12:13:32,408 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN to state=OFFLINE, location=null, table=test, region=11d894b29c3ee7cc1e5281c570908571 to restore RIT 2024-11-19T12:13:32,408 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN to state=OFFLINE, location=null, table=test, region=24b360a2b049dbc9c784cdd687321115 to restore RIT 2024-11-19T12:13:32,408 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN to state=OFFLINE, location=null, table=test, region=ab050043367e9fb30539590d5f965d47 to restore RIT 2024-11-19T12:13:32,408 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN to state=OFFLINE, location=null, table=test, region=5bc8ba883b6bd714e90daec2cb61209e to restore RIT 2024-11-19T12:13:32,408 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN to state=OFFLINE, location=null, table=test, region=f9224ed53fa7acfead92d4143c8626ff to restore RIT 2024-11-19T12:13:32,409 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018442409 2024-11-19T12:13:32,409 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:32,409 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,412 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:32,413 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:32,413 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9224ed53fa7acfead92d4143c8626ff, server=351a98347401,45367,1732018404555 2024-11-19T12:13:32,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,414 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9224ed53fa7acfead92d4143c8626ff, server=351a98347401,45367,1732018404555 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,441 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:32,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,442 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,443 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,443 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,443 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,443 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,443 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,444 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,444 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,444 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,444 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,445 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,445 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:32,445 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018472445 2024-11-19T12:13:32,445 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018532445 2024-11-19T12:13:32,445 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 36 msec 2024-11-19T12:13:32,445 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:32,446 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,446 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:32,446 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:32,446 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:32,446 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:32,446 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:32,446 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:32,447 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:32,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:32,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,452 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,453 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:32,454 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:32,454 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:32,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:32,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:32,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:32,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN 2024-11-19T12:13:32,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8bfccd6e991fb7a90ce5bcadc0a3ef5, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11d894b29c3ee7cc1e5281c570908571, server=351a98347401,45367,1732018404555 2024-11-19T12:13:32,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24b360a2b049dbc9c784cdd687321115, server=351a98347401,45367,1732018404555 2024-11-19T12:13:32,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab050043367e9fb30539590d5f965d47, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,458 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9224ed53fa7acfead92d4143c8626ff, server=351a98347401,45367,1732018404555 2024-11-19T12:13:32,458 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,458 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,458 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,458 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,458 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,458 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,458 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,458 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,458 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,458 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,459 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,459 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,459 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,459 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,459 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,459 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:32,459 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,459 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN 2024-11-19T12:13:32,459 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,459 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8bfccd6e991fb7a90ce5bcadc0a3ef5, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,459 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11d894b29c3ee7cc1e5281c570908571, server=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,459 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24b360a2b049dbc9c784cdd687321115, server=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,459 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,459 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab050043367e9fb30539590d5f965d47, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,459 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure f9224ed53fa7acfead92d4143c8626ff, server=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,459 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-19T12:13:32,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:32,460 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:32,461 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:32,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,469 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,469 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,469 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,470 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN to state=OFFLINE, location=null, table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 to restore RIT 2024-11-19T12:13:32,470 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN to state=OFFLINE, location=null, table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5 to restore RIT 2024-11-19T12:13:32,470 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN to state=OFFLINE, location=null, table=test, region=b859464d4ffda6f2df23654fb949364f to restore RIT 2024-11-19T12:13:32,470 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN to state=OFFLINE, location=null, table=test, region=11d894b29c3ee7cc1e5281c570908571 to restore RIT 2024-11-19T12:13:32,470 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN to state=OFFLINE, location=null, table=test, region=24b360a2b049dbc9c784cdd687321115 to restore RIT 2024-11-19T12:13:32,470 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN to state=OFFLINE, location=null, table=test, region=ab050043367e9fb30539590d5f965d47 to restore RIT 2024-11-19T12:13:32,470 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN to state=OFFLINE, location=null, table=test, region=5bc8ba883b6bd714e90daec2cb61209e to restore RIT 2024-11-19T12:13:32,470 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN to state=OFFLINE, location=null, table=test, region=f9224ed53fa7acfead92d4143c8626ff to restore RIT 2024-11-19T12:13:32,471 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018442470 2024-11-19T12:13:32,471 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:32,471 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,473 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:32,477 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:32,477 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab050043367e9fb30539590d5f965d47, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,477 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab050043367e9fb30539590d5f965d47, server=351a98347401,36275,1732018404438 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:32,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,484 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,485 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,486 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,486 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,486 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,486 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,487 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,487 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:32,487 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018472487 2024-11-19T12:13:32,487 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018532487 2024-11-19T12:13:32,487 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 16 msec 2024-11-19T12:13:32,487 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:32,487 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,487 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:32,487 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:32,488 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:32,488 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:32,488 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:32,488 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:32,489 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:32,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:32,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,496 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,496 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:32,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:32,498 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:32,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:32,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:32,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,501 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:32,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN 2024-11-19T12:13:32,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8bfccd6e991fb7a90ce5bcadc0a3ef5, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11d894b29c3ee7cc1e5281c570908571, server=351a98347401,45367,1732018404555 2024-11-19T12:13:32,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24b360a2b049dbc9c784cdd687321115, server=351a98347401,45367,1732018404555 2024-11-19T12:13:32,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,502 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab050043367e9fb30539590d5f965d47, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,502 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,502 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,502 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,502 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,502 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,502 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,502 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,502 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,503 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,503 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,503 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,503 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,503 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,503 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:32,503 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,503 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN 2024-11-19T12:13:32,503 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,503 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8bfccd6e991fb7a90ce5bcadc0a3ef5, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,503 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11d894b29c3ee7cc1e5281c570908571, server=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,503 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24b360a2b049dbc9c784cdd687321115, server=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,503 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,504 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab050043367e9fb30539590d5f965d47, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,504 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-11-19T12:13:32,505 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:32,505 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:32,505 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:32,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,518 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,518 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,518 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,519 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN to state=OFFLINE, location=null, table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 to restore RIT 2024-11-19T12:13:32,519 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN to state=OFFLINE, location=null, table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5 to restore RIT 2024-11-19T12:13:32,519 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN to state=OFFLINE, location=null, table=test, region=b859464d4ffda6f2df23654fb949364f to restore RIT 2024-11-19T12:13:32,519 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN to state=OFFLINE, location=null, table=test, region=11d894b29c3ee7cc1e5281c570908571 to restore RIT 2024-11-19T12:13:32,519 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN to state=OFFLINE, location=null, table=test, region=24b360a2b049dbc9c784cdd687321115 to restore RIT 2024-11-19T12:13:32,520 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN to state=OFFLINE, location=null, table=test, region=ab050043367e9fb30539590d5f965d47 to restore RIT 2024-11-19T12:13:32,520 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN to state=OFFLINE, location=null, table=test, region=5bc8ba883b6bd714e90daec2cb61209e to restore RIT 2024-11-19T12:13:32,520 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN to state=OFFLINE, location=null, table=test, region=f9224ed53fa7acfead92d4143c8626ff to restore RIT 2024-11-19T12:13:32,520 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018442520 2024-11-19T12:13:32,520 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:32,520 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,522 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:32,524 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:32,524 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,524 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:32,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,535 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,535 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,535 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,535 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,535 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,538 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:32,538 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018472538 2024-11-19T12:13:32,538 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018532538 2024-11-19T12:13:32,538 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 18 msec 2024-11-19T12:13:32,538 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:32,538 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,538 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:32,539 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:32,539 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:32,539 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:32,539 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:32,539 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:32,540 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:32,545 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:32,545 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,545 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,545 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,545 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,545 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,546 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,546 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,546 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,546 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,546 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,546 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,546 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,546 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,546 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,546 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,546 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,546 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,546 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,546 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,546 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:32,548 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:32,548 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:32,551 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:32,551 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,551 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,551 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:32,551 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,551 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,551 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,551 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,551 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,551 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,551 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,551 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,552 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,552 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:32,552 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN 2024-11-19T12:13:32,552 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,552 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8bfccd6e991fb7a90ce5bcadc0a3ef5, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,552 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11d894b29c3ee7cc1e5281c570908571, server=351a98347401,45367,1732018404555 2024-11-19T12:13:32,552 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24b360a2b049dbc9c784cdd687321115, server=351a98347401,45367,1732018404555 2024-11-19T12:13:32,552 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,552 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,552 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,552 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,552 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,552 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,552 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,552 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,552 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,552 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,552 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,552 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,553 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,553 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,553 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,553 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,553 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,553 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,553 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,553 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,553 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,553 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:32,553 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,553 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN 2024-11-19T12:13:32,553 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,553 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8bfccd6e991fb7a90ce5bcadc0a3ef5, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,553 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11d894b29c3ee7cc1e5281c570908571, server=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,553 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24b360a2b049dbc9c784cdd687321115, server=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,553 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,554 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-19T12:13:32,555 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:32,555 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:32,555 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:32,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,563 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,563 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,563 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,564 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN to state=OFFLINE, location=null, table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 to restore RIT 2024-11-19T12:13:32,564 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN to state=OFFLINE, location=null, table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5 to restore RIT 2024-11-19T12:13:32,564 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN to state=OFFLINE, location=null, table=test, region=b859464d4ffda6f2df23654fb949364f to restore RIT 2024-11-19T12:13:32,564 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN to state=OFFLINE, location=null, table=test, region=11d894b29c3ee7cc1e5281c570908571 to restore RIT 2024-11-19T12:13:32,564 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN to state=OFFLINE, location=null, table=test, region=24b360a2b049dbc9c784cdd687321115 to restore RIT 2024-11-19T12:13:32,564 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN to state=OFFLINE, location=null, table=test, region=ab050043367e9fb30539590d5f965d47 to restore RIT 2024-11-19T12:13:32,564 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN to state=OFFLINE, location=null, table=test, region=5bc8ba883b6bd714e90daec2cb61209e to restore RIT 2024-11-19T12:13:32,565 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN to state=OFFLINE, location=null, table=test, region=f9224ed53fa7acfead92d4143c8626ff to restore RIT 2024-11-19T12:13:32,565 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018442565 2024-11-19T12:13:32,565 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:32,565 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,566 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:32,567 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:32,568 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24b360a2b049dbc9c784cdd687321115, server=351a98347401,45367,1732018404555 2024-11-19T12:13:32,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,568 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24b360a2b049dbc9c784cdd687321115, server=351a98347401,45367,1732018404555 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:32,573 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,574 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,574 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,574 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,574 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,574 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,574 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,575 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,575 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,575 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,575 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,575 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,575 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,575 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,575 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,576 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,576 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,576 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,576 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:32,576 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018472576 2024-11-19T12:13:32,576 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018532576 2024-11-19T12:13:32,576 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-19T12:13:32,576 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:32,576 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,576 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:32,576 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:32,577 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:32,577 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:32,577 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:32,577 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:32,578 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:32,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:32,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,582 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,582 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,582 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,582 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,582 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,582 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,582 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,582 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:32,583 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:32,583 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:32,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:32,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:32,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,585 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8bfccd6e991fb7a90ce5bcadc0a3ef5, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11d894b29c3ee7cc1e5281c570908571, server=351a98347401,45367,1732018404555 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24b360a2b049dbc9c784cdd687321115, server=351a98347401,45367,1732018404555 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,586 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,586 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,586 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,586 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,586 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,586 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,587 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,587 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,587 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,587 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:32,587 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,587 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN 2024-11-19T12:13:32,587 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,587 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8bfccd6e991fb7a90ce5bcadc0a3ef5, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,587 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11d894b29c3ee7cc1e5281c570908571, server=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,587 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24b360a2b049dbc9c784cdd687321115, server=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,587 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-19T12:13:32,588 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:32,588 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:32,588 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:32,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,595 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,595 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,595 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,596 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN to state=OFFLINE, location=null, table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 to restore RIT 2024-11-19T12:13:32,596 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN to state=OFFLINE, location=null, table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5 to restore RIT 2024-11-19T12:13:32,596 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN to state=OFFLINE, location=null, table=test, region=b859464d4ffda6f2df23654fb949364f to restore RIT 2024-11-19T12:13:32,596 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN to state=OFFLINE, location=null, table=test, region=11d894b29c3ee7cc1e5281c570908571 to restore RIT 2024-11-19T12:13:32,596 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN to state=OFFLINE, location=null, table=test, region=24b360a2b049dbc9c784cdd687321115 to restore RIT 2024-11-19T12:13:32,596 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN to state=OFFLINE, location=null, table=test, region=ab050043367e9fb30539590d5f965d47 to restore RIT 2024-11-19T12:13:32,597 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN to state=OFFLINE, location=null, table=test, region=5bc8ba883b6bd714e90daec2cb61209e to restore RIT 2024-11-19T12:13:32,597 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN to state=OFFLINE, location=null, table=test, region=f9224ed53fa7acfead92d4143c8626ff to restore RIT 2024-11-19T12:13:32,597 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018442597 2024-11-19T12:13:32,597 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:32,597 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,598 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:32,603 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,604 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:32,604 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11d894b29c3ee7cc1e5281c570908571, server=351a98347401,45367,1732018404555 2024-11-19T12:13:32,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,604 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11d894b29c3ee7cc1e5281c570908571, server=351a98347401,45367,1732018404555 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,604 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,605 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:32,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,606 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,607 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,609 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:32,609 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018472609 2024-11-19T12:13:32,609 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018532609 2024-11-19T12:13:32,609 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-19T12:13:32,609 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:32,609 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,609 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:32,609 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:32,610 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:32,610 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:32,610 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:32,610 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:32,611 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:32,614 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:32,615 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,615 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,615 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,615 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,615 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,615 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,615 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,615 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,615 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,615 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,615 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,615 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,615 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,615 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,615 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,615 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,615 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,616 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:32,617 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:32,617 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:32,619 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:32,619 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,619 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,619 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:32,619 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,619 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,619 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,619 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,619 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,619 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,619 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,619 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,619 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,619 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:32,620 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN 2024-11-19T12:13:32,620 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,620 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8bfccd6e991fb7a90ce5bcadc0a3ef5, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,620 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11d894b29c3ee7cc1e5281c570908571, server=351a98347401,45367,1732018404555 2024-11-19T12:13:32,620 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,620 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,620 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,620 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,620 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,620 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,620 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,620 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,620 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,620 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,620 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,620 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,620 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,620 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,620 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,620 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,620 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,620 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,620 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,621 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,621 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:32,621 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,621 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN 2024-11-19T12:13:32,621 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,621 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8bfccd6e991fb7a90ce5bcadc0a3ef5, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,621 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11d894b29c3ee7cc1e5281c570908571, server=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,621 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-19T12:13:32,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:32,622 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:32,622 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:32,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,634 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,634 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,635 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,635 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,636 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN to state=OFFLINE, location=null, table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 to restore RIT 2024-11-19T12:13:32,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN to state=OFFLINE, location=null, table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5 to restore RIT 2024-11-19T12:13:32,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN to state=OFFLINE, location=null, table=test, region=b859464d4ffda6f2df23654fb949364f to restore RIT 2024-11-19T12:13:32,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN to state=OFFLINE, location=null, table=test, region=11d894b29c3ee7cc1e5281c570908571 to restore RIT 2024-11-19T12:13:32,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN to state=OFFLINE, location=null, table=test, region=24b360a2b049dbc9c784cdd687321115 to restore RIT 2024-11-19T12:13:32,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN to state=OFFLINE, location=null, table=test, region=ab050043367e9fb30539590d5f965d47 to restore RIT 2024-11-19T12:13:32,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN to state=OFFLINE, location=null, table=test, region=5bc8ba883b6bd714e90daec2cb61209e to restore RIT 2024-11-19T12:13:32,636 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN to state=OFFLINE, location=null, table=test, region=f9224ed53fa7acfead92d4143c8626ff to restore RIT 2024-11-19T12:13:32,637 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018442637 2024-11-19T12:13:32,637 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:32,637 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,638 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:32,639 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:32,639 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8bfccd6e991fb7a90ce5bcadc0a3ef5, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,639 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8bfccd6e991fb7a90ce5bcadc0a3ef5, server=351a98347401,36275,1732018404438 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,644 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,646 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,646 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,646 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:32,646 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,650 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:32,650 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018472650 2024-11-19T12:13:32,650 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018532650 2024-11-19T12:13:32,650 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 13 msec 2024-11-19T12:13:32,650 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:32,650 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,650 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:32,650 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:32,651 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:32,651 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:32,651 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:32,651 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:32,652 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:32,655 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:32,655 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,655 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,655 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,655 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,655 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,655 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,655 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,656 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,656 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:32,657 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:32,657 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:32,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:32,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:32,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,659 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:32,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN 2024-11-19T12:13:32,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8bfccd6e991fb7a90ce5bcadc0a3ef5, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,660 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,660 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,660 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,660 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,660 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,660 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,660 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,660 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,660 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,660 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,661 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,661 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,661 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,661 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,661 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,661 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,661 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,661 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,661 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,661 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,661 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:32,661 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,661 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN 2024-11-19T12:13:32,661 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,661 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8bfccd6e991fb7a90ce5bcadc0a3ef5, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,661 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-19T12:13:32,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:32,662 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:32,663 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:32,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,669 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN to state=OFFLINE, location=null, table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 to restore RIT 2024-11-19T12:13:32,670 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN to state=OFFLINE, location=null, table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5 to restore RIT 2024-11-19T12:13:32,670 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN to state=OFFLINE, location=null, table=test, region=b859464d4ffda6f2df23654fb949364f to restore RIT 2024-11-19T12:13:32,670 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN to state=OFFLINE, location=null, table=test, region=11d894b29c3ee7cc1e5281c570908571 to restore RIT 2024-11-19T12:13:32,670 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN to state=OFFLINE, location=null, table=test, region=24b360a2b049dbc9c784cdd687321115 to restore RIT 2024-11-19T12:13:32,670 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN to state=OFFLINE, location=null, table=test, region=ab050043367e9fb30539590d5f965d47 to restore RIT 2024-11-19T12:13:32,670 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN to state=OFFLINE, location=null, table=test, region=5bc8ba883b6bd714e90daec2cb61209e to restore RIT 2024-11-19T12:13:32,670 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN to state=OFFLINE, location=null, table=test, region=f9224ed53fa7acfead92d4143c8626ff to restore RIT 2024-11-19T12:13:32,670 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018442670 2024-11-19T12:13:32,670 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:32,670 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,672 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:32,673 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:32,673 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,673 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, server=351a98347401,36275,1732018404438 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,676 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,676 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:32,677 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,678 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,679 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,680 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:32,680 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018472680 2024-11-19T12:13:32,680 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018532680 2024-11-19T12:13:32,680 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-19T12:13:32,680 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:32,680 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,680 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:32,680 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:32,680 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:32,680 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:32,680 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:32,681 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:32,681 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:32,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:32,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,684 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,685 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,685 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,685 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,685 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:32,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:32,686 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:32,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:32,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:32,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:32,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN 2024-11-19T12:13:32,688 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,688 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,689 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,689 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,689 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,689 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,689 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,689 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,689 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,689 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,689 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,689 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,689 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,689 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,689 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:32,689 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,689 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN 2024-11-19T12:13:32,689 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,689 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-19T12:13:32,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:32,690 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:32,691 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:32,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,700 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,700 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,700 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,701 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN to state=OFFLINE, location=null, table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 to restore RIT 2024-11-19T12:13:32,701 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN to state=OFFLINE, location=null, table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5 to restore RIT 2024-11-19T12:13:32,701 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN to state=OFFLINE, location=null, table=test, region=b859464d4ffda6f2df23654fb949364f to restore RIT 2024-11-19T12:13:32,701 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN to state=OFFLINE, location=null, table=test, region=11d894b29c3ee7cc1e5281c570908571 to restore RIT 2024-11-19T12:13:32,701 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN to state=OFFLINE, location=null, table=test, region=24b360a2b049dbc9c784cdd687321115 to restore RIT 2024-11-19T12:13:32,701 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN to state=OFFLINE, location=null, table=test, region=ab050043367e9fb30539590d5f965d47 to restore RIT 2024-11-19T12:13:32,702 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN to state=OFFLINE, location=null, table=test, region=5bc8ba883b6bd714e90daec2cb61209e to restore RIT 2024-11-19T12:13:32,702 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN to state=OFFLINE, location=null, table=test, region=f9224ed53fa7acfead92d4143c8626ff to restore RIT 2024-11-19T12:13:32,702 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018442702 2024-11-19T12:13:32,702 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:32,702 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,704 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:32,705 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:32,705 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN 2024-11-19T12:13:32,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,705 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,711 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,711 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,711 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:32,711 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,711 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,711 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,712 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,713 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,713 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,713 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,713 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,713 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,713 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,714 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,714 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,714 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,714 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:32,714 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018472714 2024-11-19T12:13:32,714 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018532714 2024-11-19T12:13:32,714 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 12 msec 2024-11-19T12:13:32,714 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:32,714 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,714 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:32,715 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:32,715 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:32,715 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:32,715 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:32,715 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:32,716 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:32,719 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:32,719 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,720 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,720 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,720 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,720 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,720 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,720 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,720 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,720 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,720 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,720 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,720 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,720 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,720 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,721 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:32,721 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:32,722 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:32,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:32,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:32,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,723 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,724 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,724 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,724 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,724 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,724 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,724 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,724 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,724 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:32,724 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN 2024-11-19T12:13:32,724 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,724 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,724 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,724 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,724 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,724 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,724 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,724 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,725 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,725 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,725 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,725 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,725 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,725 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,725 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,725 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,725 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,725 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,725 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,725 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,725 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:32,725 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,725 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN 2024-11-19T12:13:32,726 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-19T12:13:32,727 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:32,727 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:32,727 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:32,738 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,739 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,739 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,739 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,739 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,740 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN to state=OFFLINE, location=null, table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 to restore RIT 2024-11-19T12:13:32,740 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN to state=OFFLINE, location=null, table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5 to restore RIT 2024-11-19T12:13:32,740 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN to state=OFFLINE, location=null, table=test, region=b859464d4ffda6f2df23654fb949364f to restore RIT 2024-11-19T12:13:32,741 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN to state=OFFLINE, location=null, table=test, region=11d894b29c3ee7cc1e5281c570908571 to restore RIT 2024-11-19T12:13:32,741 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN to state=OFFLINE, location=null, table=test, region=24b360a2b049dbc9c784cdd687321115 to restore RIT 2024-11-19T12:13:32,741 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN to state=OFFLINE, location=null, table=test, region=ab050043367e9fb30539590d5f965d47 to restore RIT 2024-11-19T12:13:32,741 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN to state=OFFLINE, location=null, table=test, region=5bc8ba883b6bd714e90daec2cb61209e to restore RIT 2024-11-19T12:13:32,741 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f9224ed53fa7acfead92d4143c8626ff, ASSIGN to state=OFFLINE, location=null, table=test, region=f9224ed53fa7acfead92d4143c8626ff to restore RIT 2024-11-19T12:13:32,741 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018442741 2024-11-19T12:13:32,741 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:32,742 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,743 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:32,747 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:32,748 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:32,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,748 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:32,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,751 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,752 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,753 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:32,753 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018472753 2024-11-19T12:13:32,753 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018532753 2024-11-19T12:13:32,753 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-19T12:13:32,753 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:32,753 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,753 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:32,753 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:32,754 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:32,754 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:32,754 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:32,754 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:32,755 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:32,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:32,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,758 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:32,759 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:32,759 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,760 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,760 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,760 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,761 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,761 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,761 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,761 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,761 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,761 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,761 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,761 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,761 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,761 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN 2024-11-19T12:13:32,761 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-19T12:13:32,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:32,762 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:32,762 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:32,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,774 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,774 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,774 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,775 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN to state=OFFLINE, location=null, table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 to restore RIT 2024-11-19T12:13:32,775 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN to state=OFFLINE, location=null, table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5 to restore RIT 2024-11-19T12:13:32,775 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN to state=OFFLINE, location=null, table=test, region=b859464d4ffda6f2df23654fb949364f to restore RIT 2024-11-19T12:13:32,775 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN to state=OFFLINE, location=null, table=test, region=11d894b29c3ee7cc1e5281c570908571 to restore RIT 2024-11-19T12:13:32,775 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN to state=OFFLINE, location=null, table=test, region=24b360a2b049dbc9c784cdd687321115 to restore RIT 2024-11-19T12:13:32,775 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN to state=OFFLINE, location=null, table=test, region=ab050043367e9fb30539590d5f965d47 to restore RIT 2024-11-19T12:13:32,775 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=5bc8ba883b6bd714e90daec2cb61209e, ASSIGN to state=OFFLINE, location=null, table=test, region=5bc8ba883b6bd714e90daec2cb61209e to restore RIT 2024-11-19T12:13:32,779 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018442779 2024-11-19T12:13:32,779 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:32,779 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,780 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:32,782 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:32,782 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,782 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:32,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,789 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:32,789 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018472789 2024-11-19T12:13:32,789 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018532789 2024-11-19T12:13:32,789 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-19T12:13:32,789 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:32,789 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,789 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:32,790 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:32,790 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:32,790 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:32,790 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:32,790 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:32,791 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:32,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:32,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:32,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:32,795 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:32,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:32,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:32,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,797 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,797 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,797 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,797 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,797 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,797 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,797 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,797 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,797 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,797 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,797 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,797 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,797 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,797 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN 2024-11-19T12:13:32,797 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-19T12:13:32,798 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:32,798 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:32,799 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:32,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,805 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,805 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,805 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,806 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN to state=OFFLINE, location=null, table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 to restore RIT 2024-11-19T12:13:32,806 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN to state=OFFLINE, location=null, table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5 to restore RIT 2024-11-19T12:13:32,806 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN to state=OFFLINE, location=null, table=test, region=b859464d4ffda6f2df23654fb949364f to restore RIT 2024-11-19T12:13:32,806 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN to state=OFFLINE, location=null, table=test, region=11d894b29c3ee7cc1e5281c570908571 to restore RIT 2024-11-19T12:13:32,806 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN to state=OFFLINE, location=null, table=test, region=24b360a2b049dbc9c784cdd687321115 to restore RIT 2024-11-19T12:13:32,807 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab050043367e9fb30539590d5f965d47, ASSIGN to state=OFFLINE, location=null, table=test, region=ab050043367e9fb30539590d5f965d47 to restore RIT 2024-11-19T12:13:32,807 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018442807 2024-11-19T12:13:32,807 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:32,807 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,808 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:32,809 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:32,809 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,809 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,813 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,814 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:32,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,815 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,816 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,818 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,818 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:32,818 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018472818 2024-11-19T12:13:32,818 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018532818 2024-11-19T12:13:32,818 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-19T12:13:32,818 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:32,818 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,818 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:32,818 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:32,818 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:32,818 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:32,818 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:32,819 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:32,820 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:32,822 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:32,822 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,822 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,822 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,822 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,822 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,822 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,822 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,822 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,822 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,822 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,822 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,822 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:32,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:32,823 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:32,825 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:32,825 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,825 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,825 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:32,825 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,825 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,825 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,825 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,825 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,825 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,825 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,825 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,825 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,826 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,826 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,826 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,826 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,826 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,826 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,826 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,826 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,826 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,826 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,826 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN 2024-11-19T12:13:32,826 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-19T12:13:32,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:32,828 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:32,828 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:32,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,837 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,837 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,837 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,838 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN to state=OFFLINE, location=null, table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 to restore RIT 2024-11-19T12:13:32,838 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN to state=OFFLINE, location=null, table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5 to restore RIT 2024-11-19T12:13:32,838 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN to state=OFFLINE, location=null, table=test, region=b859464d4ffda6f2df23654fb949364f to restore RIT 2024-11-19T12:13:32,838 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN to state=OFFLINE, location=null, table=test, region=11d894b29c3ee7cc1e5281c570908571 to restore RIT 2024-11-19T12:13:32,838 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=24b360a2b049dbc9c784cdd687321115, ASSIGN to state=OFFLINE, location=null, table=test, region=24b360a2b049dbc9c784cdd687321115 to restore RIT 2024-11-19T12:13:32,838 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018442838 2024-11-19T12:13:32,838 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:32,838 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,840 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:32,842 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:32,842 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,842 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:32,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,847 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:32,847 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018472847 2024-11-19T12:13:32,847 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018532847 2024-11-19T12:13:32,847 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-19T12:13:32,847 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:32,847 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,847 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:32,847 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:32,847 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:32,847 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:32,847 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:32,847 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:32,848 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:32,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:32,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,850 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:32,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:32,851 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:32,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:32,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:32,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,852 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,852 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,852 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,853 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,853 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,853 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,853 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,853 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,853 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,853 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN 2024-11-19T12:13:32,853 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-19T12:13:32,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:32,854 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:32,854 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:32,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,866 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,866 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,866 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,866 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN to state=OFFLINE, location=null, table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 to restore RIT 2024-11-19T12:13:32,867 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN to state=OFFLINE, location=null, table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5 to restore RIT 2024-11-19T12:13:32,867 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN to state=OFFLINE, location=null, table=test, region=b859464d4ffda6f2df23654fb949364f to restore RIT 2024-11-19T12:13:32,867 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=11d894b29c3ee7cc1e5281c570908571, ASSIGN to state=OFFLINE, location=null, table=test, region=11d894b29c3ee7cc1e5281c570908571 to restore RIT 2024-11-19T12:13:32,867 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018442867 2024-11-19T12:13:32,868 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:32,868 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,869 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:32,870 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:32,870 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,870 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:32,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,878 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:32,878 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018472878 2024-11-19T12:13:32,878 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018532878 2024-11-19T12:13:32,878 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-19T12:13:32,878 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:32,878 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,878 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:32,878 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:32,879 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:32,879 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:32,879 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:32,879 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:32,880 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:32,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:32,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,882 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,883 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,883 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,883 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,883 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,883 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,883 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,883 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:32,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:32,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:32,885 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:32,885 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,885 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,885 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:32,885 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,885 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,885 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,885 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,885 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,885 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,885 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,885 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,885 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,885 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,886 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,886 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,886 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,886 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,886 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,886 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,886 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,886 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,886 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN 2024-11-19T12:13:32,886 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-19T12:13:32,887 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:32,887 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:32,887 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:32,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,900 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,900 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,900 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,901 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN to state=OFFLINE, location=null, table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 to restore RIT 2024-11-19T12:13:32,901 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN to state=OFFLINE, location=null, table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5 to restore RIT 2024-11-19T12:13:32,901 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b859464d4ffda6f2df23654fb949364f, ASSIGN to state=OFFLINE, location=null, table=test, region=b859464d4ffda6f2df23654fb949364f to restore RIT 2024-11-19T12:13:32,901 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018442901 2024-11-19T12:13:32,901 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:32,901 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,903 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:32,904 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:32,904 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,905 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:32,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,907 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,908 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,909 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,909 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:32,909 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018472909 2024-11-19T12:13:32,909 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018532909 2024-11-19T12:13:32,910 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-19T12:13:32,910 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:32,910 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,910 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:32,910 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:32,910 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:32,910 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:32,910 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:32,910 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:32,911 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:32,913 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:32,913 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,913 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,913 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,914 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,914 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,914 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,914 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,914 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:32,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:32,915 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:32,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:32,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:32,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,916 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,916 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,916 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,916 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,917 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,917 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,917 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN 2024-11-19T12:13:32,917 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-19T12:13:32,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:32,918 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:32,918 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:32,945 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,946 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,958 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,958 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,958 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,958 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN to state=OFFLINE, location=null, table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 to restore RIT 2024-11-19T12:13:32,958 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, ASSIGN to state=OFFLINE, location=null, table=test, region=d8bfccd6e991fb7a90ce5bcadc0a3ef5 to restore RIT 2024-11-19T12:13:32,959 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018442959 2024-11-19T12:13:32,959 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:32,959 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,960 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:32,962 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:32,962 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,962 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,964 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,964 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:32,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,965 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,966 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:32,967 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:32,967 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:32,967 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018472967 2024-11-19T12:13:32,967 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018532967 2024-11-19T12:13:32,967 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-19T12:13:32,967 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:32,967 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:32,967 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:32,967 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:32,967 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:32,967 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:32,967 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:32,968 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:32,968 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:32,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:32,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:32,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:32,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:32,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:32,970 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:32,971 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:32,971 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:32,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:32,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:32,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:32,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 2024-11-19T12:13:32,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:32,972 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,972 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,972 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,972 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,972 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,972 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,972 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,972 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:32,972 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:32,972 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN 2024-11-19T12:13:32,972 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-19T12:13:32,973 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:32,973 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:32,973 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:32,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:32,995 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,995 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,995 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:32,996 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, ASSIGN to state=OFFLINE, location=null, table=test, region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 to restore RIT 2024-11-19T12:13:32,996 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018442996 2024-11-19T12:13:32,996 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:32,996 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:32,998 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:33,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,001 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:33,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,001 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:33,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:33,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,001 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:33,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:33,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,003 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,004 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,004 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:33,004 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018473004 2024-11-19T12:13:33,004 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018533004 2024-11-19T12:13:33,004 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-19T12:13:33,004 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:33,004 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:33,004 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:33,005 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:33,005 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:33,005 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:33,005 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:33,005 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:33,006 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:33,008 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:33,008 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:33,008 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:33,008 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:33,008 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:33,008 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:33,008 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:33,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:33,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:33,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:33,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:33,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:33,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:33,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:33,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 2024-11-19T12:13:33,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:33,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:33,010 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:33,010 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:33,010 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:33,011 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:33,011 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:33,011 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:33,011 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:33,011 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-19T12:13:33,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:33,012 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:33,012 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:33,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:33,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:33,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:33,024 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:33,024 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:33,024 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:33,024 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018443024 2024-11-19T12:13:33,025 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:33,025 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:33,026 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:33,028 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:33,028 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:33,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:33,028 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:33,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:33,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,033 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,034 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,034 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,034 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,034 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,034 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,034 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,034 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:33,034 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018473034 2024-11-19T12:13:33,034 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018533034 2024-11-19T12:13:33,034 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-19T12:13:33,035 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:33,035 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:33,035 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:33,035 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:33,035 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:33,035 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:33,035 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:33,035 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:33,036 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:33,038 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:33,039 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:33,039 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:33,039 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:33,039 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:33,039 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:33,039 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:33,040 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:33,040 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:33,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:33,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:33,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:33,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:33,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 2024-11-19T12:13:33,041 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 2024-11-19T12:13:33,041 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:33,041 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:33,041 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:33,041 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:33,041 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:33,041 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.1732018406120, worker=351a98347401,45367,1732018404555 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:33,042 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-19T12:13:33,042 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:33,043 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:33,043 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:33,059 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:33,060 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38839 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:33,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:33,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:33,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:33,111 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:33,111 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:33,111 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:33,112 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018443112 2024-11-19T12:13:33,112 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:33,112 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:33,113 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:33,116 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:33,116 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 2024-11-19T12:13:33,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:33,116 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:33,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:33,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,119 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:33,120 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018473119 2024-11-19T12:13:33,120 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018533120 2024-11-19T12:13:33,120 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-19T12:13:33,120 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:33,120 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:33,120 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:33,120 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:33,120 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:33,120 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:33,120 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:33,120 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:33,121 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:33,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:33,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:33,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:33,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:33,123 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-19T12:13:33,123 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:33,124 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:33,124 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:33,125 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:33,125 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:33,125 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:33,125 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:33,125 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 2024-11-19T12:13:33,125 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:33,125 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:33,125 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:33,125 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:33,125 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:33,125 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-19T12:13:33,126 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:33,126 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:33,126 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:33,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:33,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:33,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:33,139 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:33,139 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:33,139 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:33,140 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018443140 2024-11-19T12:13:33,140 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:33,140 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:33,142 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:33,143 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:33,143 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:33,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:33,143 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:33,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,145 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:33,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,146 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,147 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:33,147 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018473147 2024-11-19T12:13:33,147 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018533147 2024-11-19T12:13:33,147 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-19T12:13:33,147 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:33,147 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:33,147 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:33,147 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:33,148 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:33,148 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:33,148 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:33,148 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:33,149 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:33,150 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:33,150 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:33,150 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:33,150 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-19T12:13:33,150 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:33,151 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:33,151 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:33,151 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:33,151 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:33,151 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:33,151 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:13:33,151 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:33,151 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:33,151 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:33,151 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:33,152 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-19T12:13:33,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:33,152 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:33,152 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:33,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:33,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:33,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:33,163 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:33,163 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:33,163 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:33,164 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018443164 2024-11-19T12:13:33,164 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:33,164 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:33,165 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:33,166 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:33,166 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:33,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:33,166 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:33,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:33,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,172 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:33,172 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018473172 2024-11-19T12:13:33,172 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018533172 2024-11-19T12:13:33,172 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-19T12:13:33,172 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:33,172 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:33,172 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:33,172 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:33,173 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:33,173 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:33,173 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:33,173 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:33,174 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:33,175 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:33,175 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:33,175 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-19T12:13:33,175 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:33,176 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:33,176 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:33,176 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:33,176 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:33,176 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:33,176 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:33,176 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:33,176 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:33,177 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-19T12:13:33,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:33,177 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:33,177 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:33,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:33,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:33,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:33,184 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:33,184 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:33,184 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:33,185 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018443185 2024-11-19T12:13:33,185 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:33,185 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:33,187 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:33,188 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:33,188 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:33,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:33,188 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:33,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:33,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,191 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,192 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:33,192 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018473192 2024-11-19T12:13:33,192 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018533192 2024-11-19T12:13:33,192 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-19T12:13:33,192 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:33,192 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:33,192 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:33,192 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:33,192 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:33,192 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:33,192 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:33,192 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:33,193 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:33,195 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:33,195 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-19T12:13:33,195 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:33,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:33,196 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:33,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:33,197 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 2024-11-19T12:13:33,197 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:33,197 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 351a98347401%2C35543%2C1732018404609.meta.1732018406567.meta, worker=351a98347401,36275,1732018404438 didn't hold the lock before restarting, skip acquiring lock. 2024-11-19T12:13:33,197 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-19T12:13:33,198 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:33,198 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:33,198 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:33,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:33,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:33,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:33,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:33,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:33,205 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:33,206 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018443206 2024-11-19T12:13:33,206 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:33,206 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:33,207 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:33,208 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-19T12:13:33,208 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:33,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:33,208 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:33,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:33,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,215 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,215 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:33,215 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018473215 2024-11-19T12:13:33,216 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018533216 2024-11-19T12:13:33,216 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-19T12:13:33,216 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-19T12:13:33,216 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:33,216 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:33,216 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:33,216 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-19T12:13:33,216 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-19T12:13:33,216 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-19T12:13:33,216 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:13:33,217 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:13:33,218 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-19T12:13:33,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:13:33,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-19T12:13:33,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7382ad40044282a358da8130ce023367, REOPEN/MOVE 2024-11-19T12:13:33,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true 2024-11-19T12:13:33,219 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-19T12:13:33,220 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-19T12:13:33,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=65 2024-11-19T12:13:33,220 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=351a98347401,36275,1732018404438, table=hbase:meta, region=1588230740 2024-11-19T12:13:33,220 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 351a98347401,36275,1732018404438, state=OPEN 2024-11-19T12:13:33,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:33,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:33,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:13:33,226 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:33,226 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:33,226 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:13:33,227 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732018443227 2024-11-19T12:13:33,227 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T12:13:33,227 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-19T12:13:33,229 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-19T12:13:33,230 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=69, state=RUNNABLE, hasLock=true; OpenRegionProcedure b859464d4ffda6f2df23654fb949364f, server=351a98347401,36275,1732018404438:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 351a98347401,35543,1732018404609, splitWal=true, meta=true exec-time=2.6960 sec 2024-11-19T12:13:33,233 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b859464d4ffda6f2df23654fb949364f, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,233 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e257775d8750f12b70161bb0b6387163, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,233 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5b9b9ba699dcd89ef4b3398f2beb2455, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fbade10b0e1d2be0489df2d6c3fcf9d, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f9224ed53fa7acfead92d4143c8626ff, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a78d1de57ec917279b938f869bdbe10d, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=125c93438c95e8e5f1b8be66c59d7cf9, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7382ad40044282a358da8130ce023367, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=5 2024-11-19T12:13:33,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab050043367e9fb30539590d5f965d47, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19add541d34392b48fdd488f1c19c182, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a329354a61adc5325fa166bf564b84eb, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5bc8ba883b6bd714e90daec2cb61209e, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=adc3abaf30d887a0d57a7730313a62b4, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef4298fd6621c8e7facf42422bd80be4, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,234 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d8bfccd6e991fb7a90ce5bcadc0a3ef5, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=963c42e1179165051d86c1560642c124, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca6b3c7ec060f344fff683e91e796d33, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24b360a2b049dbc9c784cdd687321115, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=797a72fe833580fc974f887653d7763a, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2ec97ffd6b2cc998be397eec7fc0a0bc, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b6d3f4b598ab015d9c62b67f0bac50f, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e85bee36dde7095f8d08f860a32613d7, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11d894b29c3ee7cc1e5281c570908571, regionState=OPENING, lastHost=351a98347401,35543,1732018404609, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=af2a9e847c9e4d4812af432c436c7ac0, regionState=OPEN, lastHost=351a98347401,36275,1732018404438, regionLocation=351a98347401,36275,1732018404438, openSeqNum=2 2024-11-19T12:13:33,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fea5432239dac4bfb1b607057a95e34, regionState=OPEN, lastHost=351a98347401,45367,1732018404555, regionLocation=351a98347401,45367,1732018404555, openSeqNum=2 2024-11-19T12:13:33,235 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-19T12:13:33,235 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732018473235 2024-11-19T12:13:33,236 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732018533235 2024-11-19T12:13:33,236 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-19T12:13:33,251 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=350 (was 383), OpenFileDescriptor=620 (was 642), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=329 (was 314) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=13362 (was 13425) 2024-11-19T12:13:33,251 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T12:13:33,251 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T12:13:33,252 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:13:33,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:13:33,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:13:33,252 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:13:33,252 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T12:13:33,252 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=654212325, stopped=false 2024-11-19T12:13:33,253 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=351a98347401,38839,1732018403620 2024-11-19T12:13:33,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T12:13:33,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T12:13:33,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:33,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T12:13:33,266 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:33,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:33,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:33,266 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T12:13:33,266 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:13:33,266 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:13:33,266 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:13:33,266 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:13:33,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:13:33,267 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '351a98347401,36275,1732018404438' ***** 2024-11-19T12:13:33,267 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T12:13:33,267 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '351a98347401,45367,1732018404555' ***** 2024-11-19T12:13:33,268 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T12:13:33,268 INFO [RS:0;351a98347401:36275 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T12:13:33,268 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T12:13:33,268 INFO [RS:1;351a98347401:45367 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T12:13:33,268 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T12:13:33,268 INFO [RS:1;351a98347401:45367 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T12:13:33,268 INFO [RS:0;351a98347401:36275 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T12:13:33,268 INFO [RS:1;351a98347401:45367 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T12:13:33,268 INFO [RS:0;351a98347401:36275 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T12:13:33,268 INFO [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(3091): Received CLOSE for 125c93438c95e8e5f1b8be66c59d7cf9 2024-11-19T12:13:33,268 INFO [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(3091): Received CLOSE for 797a72fe833580fc974f887653d7763a 2024-11-19T12:13:33,269 INFO [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(3091): Received CLOSE for a329354a61adc5325fa166bf564b84eb 2024-11-19T12:13:33,269 INFO [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(3091): Received CLOSE for 8fbade10b0e1d2be0489df2d6c3fcf9d 2024-11-19T12:13:33,269 INFO [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(3091): Received CLOSE for 24b360a2b049dbc9c784cdd687321115 2024-11-19T12:13:33,269 INFO [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(3091): Received CLOSE for 5bc8ba883b6bd714e90daec2cb61209e 2024-11-19T12:13:33,269 INFO [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(3091): Received CLOSE for ca6b3c7ec060f344fff683e91e796d33 2024-11-19T12:13:33,269 INFO [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(3091): Received CLOSE for 4fea5432239dac4bfb1b607057a95e34 2024-11-19T12:13:33,269 INFO [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(3091): Received CLOSE for ab050043367e9fb30539590d5f965d47 2024-11-19T12:13:33,270 INFO [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(3091): Received CLOSE for f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:33,270 INFO [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(3091): Received CLOSE for e85bee36dde7095f8d08f860a32613d7 2024-11-19T12:13:33,270 INFO [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(3091): Received CLOSE for 2b6d3f4b598ab015d9c62b67f0bac50f 2024-11-19T12:13:33,270 INFO [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(3091): Received CLOSE for 2ec97ffd6b2cc998be397eec7fc0a0bc 2024-11-19T12:13:33,270 INFO [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(3091): Received CLOSE for af2a9e847c9e4d4812af432c436c7ac0 2024-11-19T12:13:33,270 INFO [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(3091): Received CLOSE for ef4298fd6621c8e7facf42422bd80be4 2024-11-19T12:13:33,270 INFO [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(3091): Received CLOSE for 19add541d34392b48fdd488f1c19c182 2024-11-19T12:13:33,270 INFO [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(3091): Received CLOSE for 5b9b9ba699dcd89ef4b3398f2beb2455 2024-11-19T12:13:33,270 INFO [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(3091): Received CLOSE for 11d894b29c3ee7cc1e5281c570908571 2024-11-19T12:13:33,270 INFO [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(3091): Received CLOSE for adc3abaf30d887a0d57a7730313a62b4 2024-11-19T12:13:33,270 INFO [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(3091): Received CLOSE for b859464d4ffda6f2df23654fb949364f 2024-11-19T12:13:33,270 INFO [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(959): stopping server 351a98347401,45367,1732018404555 2024-11-19T12:13:33,270 INFO [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(3091): Received CLOSE for a78d1de57ec917279b938f869bdbe10d 2024-11-19T12:13:33,270 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 797a72fe833580fc974f887653d7763a, disabling compactions & flushes 2024-11-19T12:13:33,270 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 125c93438c95e8e5f1b8be66c59d7cf9, disabling compactions & flushes 2024-11-19T12:13:33,270 INFO [RS:1;351a98347401:45367 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T12:13:33,270 INFO [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(3091): Received CLOSE for 963c42e1179165051d86c1560642c124 2024-11-19T12:13:33,270 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1732018406961.125c93438c95e8e5f1b8be66c59d7cf9. 2024-11-19T12:13:33,270 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1732018406961.797a72fe833580fc974f887653d7763a. 2024-11-19T12:13:33,270 INFO [RS:1;351a98347401:45367 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;351a98347401:45367. 2024-11-19T12:13:33,270 INFO [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(3091): Received CLOSE for d8bfccd6e991fb7a90ce5bcadc0a3ef5 2024-11-19T12:13:33,270 INFO [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(3091): Received CLOSE for e257775d8750f12b70161bb0b6387163 2024-11-19T12:13:33,270 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732018406961.125c93438c95e8e5f1b8be66c59d7cf9. 2024-11-19T12:13:33,270 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732018406961.797a72fe833580fc974f887653d7763a. 2024-11-19T12:13:33,270 INFO [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(3091): Received CLOSE for c48a9e94bd9fcbd6f8c1f6ae2b6c80d3 2024-11-19T12:13:33,270 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732018406961.125c93438c95e8e5f1b8be66c59d7cf9. after waiting 0 ms 2024-11-19T12:13:33,270 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732018406961.797a72fe833580fc974f887653d7763a. after waiting 0 ms 2024-11-19T12:13:33,270 INFO [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(3091): Received CLOSE for 7382ad40044282a358da8130ce023367 2024-11-19T12:13:33,270 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732018406961.125c93438c95e8e5f1b8be66c59d7cf9. 2024-11-19T12:13:33,270 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732018406961.797a72fe833580fc974f887653d7763a. 2024-11-19T12:13:33,270 DEBUG [RS:1;351a98347401:45367 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:13:33,270 INFO [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(959): stopping server 351a98347401,36275,1732018404438 2024-11-19T12:13:33,270 DEBUG [RS:1;351a98347401:45367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:13:33,270 INFO [RS:0;351a98347401:36275 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T12:13:33,270 INFO [RS:0;351a98347401:36275 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;351a98347401:36275. 2024-11-19T12:13:33,270 DEBUG [RS:0;351a98347401:36275 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:13:33,270 INFO [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-19T12:13:33,270 DEBUG [RS:0;351a98347401:36275 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:13:33,271 INFO [RS:0;351a98347401:36275 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T12:13:33,271 INFO [RS:0;351a98347401:36275 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T12:13:33,271 INFO [RS:0;351a98347401:36275 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T12:13:33,271 DEBUG [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(1351): Waiting on 11d894b29c3ee7cc1e5281c570908571, 125c93438c95e8e5f1b8be66c59d7cf9, 24b360a2b049dbc9c784cdd687321115, 2ec97ffd6b2cc998be397eec7fc0a0bc, 4fea5432239dac4bfb1b607057a95e34, 5b9b9ba699dcd89ef4b3398f2beb2455, 5bc8ba883b6bd714e90daec2cb61209e, 8fbade10b0e1d2be0489df2d6c3fcf9d, a329354a61adc5325fa166bf564b84eb, ca6b3c7ec060f344fff683e91e796d33, e85bee36dde7095f8d08f860a32613d7, ef4298fd6621c8e7facf42422bd80be4, f9224ed53fa7acfead92d4143c8626ff 2024-11-19T12:13:33,271 INFO [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T12:13:33,271 INFO [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-19T12:13:33,271 DEBUG [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 19add541d34392b48fdd488f1c19c182, 2b6d3f4b598ab015d9c62b67f0bac50f, 7382ad40044282a358da8130ce023367, 797a72fe833580fc974f887653d7763a, 963c42e1179165051d86c1560642c124, a78d1de57ec917279b938f869bdbe10d, ab050043367e9fb30539590d5f965d47, adc3abaf30d887a0d57a7730313a62b4, af2a9e847c9e4d4812af432c436c7ac0, b859464d4ffda6f2df23654fb949364f, c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, d8bfccd6e991fb7a90ce5bcadc0a3ef5, e257775d8750f12b70161bb0b6387163 2024-11-19T12:13:33,272 DEBUG [RS_CLOSE_META-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T12:13:33,272 INFO [RS_CLOSE_META-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T12:13:33,272 DEBUG [RS_CLOSE_META-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T12:13:33,272 DEBUG [RS_CLOSE_META-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T12:13:33,272 DEBUG [RS_CLOSE_META-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T12:13:33,273 INFO [RS_CLOSE_META-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.64 KB heapSize=5.42 KB 2024-11-19T12:13:33,276 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/125c93438c95e8e5f1b8be66c59d7cf9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,276 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/797a72fe833580fc974f887653d7763a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,277 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1732018406961.797a72fe833580fc974f887653d7763a. 2024-11-19T12:13:33,277 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 797a72fe833580fc974f887653d7763a: Waiting for close lock at 1732018413270Running coprocessor pre-close hooks at 1732018413270Disabling compacts and flushes for region at 1732018413270Disabling writes for close at 1732018413270Writing region close event to WAL at 1732018413271 (+1 ms)Running coprocessor post-close hooks at 1732018413277 (+6 ms)Closed at 1732018413277 2024-11-19T12:13:33,278 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1732018406961.797a72fe833580fc974f887653d7763a. 2024-11-19T12:13:33,278 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ab050043367e9fb30539590d5f965d47, disabling compactions & flushes 2024-11-19T12:13:33,278 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1732018406961.125c93438c95e8e5f1b8be66c59d7cf9. 2024-11-19T12:13:33,278 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1732018406961.ab050043367e9fb30539590d5f965d47. 2024-11-19T12:13:33,278 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732018406961.ab050043367e9fb30539590d5f965d47. 2024-11-19T12:13:33,278 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 125c93438c95e8e5f1b8be66c59d7cf9: Waiting for close lock at 1732018413269Running coprocessor pre-close hooks at 1732018413270 (+1 ms)Disabling compacts and flushes for region at 1732018413270Disabling writes for close at 1732018413270Writing region close event to WAL at 1732018413271 (+1 ms)Running coprocessor post-close hooks at 1732018413277 (+6 ms)Closed at 1732018413278 (+1 ms) 2024-11-19T12:13:33,278 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732018406961.ab050043367e9fb30539590d5f965d47. after waiting 0 ms 2024-11-19T12:13:33,278 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732018406961.ab050043367e9fb30539590d5f965d47. 2024-11-19T12:13:33,278 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1732018406961.125c93438c95e8e5f1b8be66c59d7cf9. 2024-11-19T12:13:33,278 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a329354a61adc5325fa166bf564b84eb, disabling compactions & flushes 2024-11-19T12:13:33,278 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1732018406961.a329354a61adc5325fa166bf564b84eb. 2024-11-19T12:13:33,278 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732018406961.a329354a61adc5325fa166bf564b84eb. 2024-11-19T12:13:33,278 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732018406961.a329354a61adc5325fa166bf564b84eb. after waiting 0 ms 2024-11-19T12:13:33,278 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732018406961.a329354a61adc5325fa166bf564b84eb. 2024-11-19T12:13:33,284 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/ab050043367e9fb30539590d5f965d47/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,284 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/a329354a61adc5325fa166bf564b84eb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,285 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1732018406961.a329354a61adc5325fa166bf564b84eb. 2024-11-19T12:13:33,285 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a329354a61adc5325fa166bf564b84eb: Waiting for close lock at 1732018413278Running coprocessor pre-close hooks at 1732018413278Disabling compacts and flushes for region at 1732018413278Disabling writes for close at 1732018413278Writing region close event to WAL at 1732018413279 (+1 ms)Running coprocessor post-close hooks at 1732018413285 (+6 ms)Closed at 1732018413285 2024-11-19T12:13:33,285 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1732018406961.ab050043367e9fb30539590d5f965d47. 2024-11-19T12:13:33,285 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ab050043367e9fb30539590d5f965d47: Waiting for close lock at 1732018413278Running coprocessor pre-close hooks at 1732018413278Disabling compacts and flushes for region at 1732018413278Disabling writes for close at 1732018413278Writing region close event to WAL at 1732018413278Running coprocessor post-close hooks at 1732018413285 (+7 ms)Closed at 1732018413285 2024-11-19T12:13:33,285 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1732018406961.a329354a61adc5325fa166bf564b84eb. 2024-11-19T12:13:33,285 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1732018406961.ab050043367e9fb30539590d5f965d47. 2024-11-19T12:13:33,285 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8fbade10b0e1d2be0489df2d6c3fcf9d, disabling compactions & flushes 2024-11-19T12:13:33,285 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1732018406961.8fbade10b0e1d2be0489df2d6c3fcf9d. 2024-11-19T12:13:33,285 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2b6d3f4b598ab015d9c62b67f0bac50f, disabling compactions & flushes 2024-11-19T12:13:33,285 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732018406961.8fbade10b0e1d2be0489df2d6c3fcf9d. 2024-11-19T12:13:33,285 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1732018406961.2b6d3f4b598ab015d9c62b67f0bac50f. 2024-11-19T12:13:33,285 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732018406961.8fbade10b0e1d2be0489df2d6c3fcf9d. after waiting 0 ms 2024-11-19T12:13:33,285 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732018406961.2b6d3f4b598ab015d9c62b67f0bac50f. 2024-11-19T12:13:33,285 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732018406961.8fbade10b0e1d2be0489df2d6c3fcf9d. 2024-11-19T12:13:33,285 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732018406961.2b6d3f4b598ab015d9c62b67f0bac50f. after waiting 0 ms 2024-11-19T12:13:33,285 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732018406961.2b6d3f4b598ab015d9c62b67f0bac50f. 2024-11-19T12:13:33,288 DEBUG [RS_CLOSE_META-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/.tmp/info/033f3aaa4cee40e6b704061229c5002c is 138, key is test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff./info:regioninfo/1732018411983/Put/seqid=0 2024-11-19T12:13:33,292 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/8fbade10b0e1d2be0489df2d6c3fcf9d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,293 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1732018406961.8fbade10b0e1d2be0489df2d6c3fcf9d. 2024-11-19T12:13:33,293 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8fbade10b0e1d2be0489df2d6c3fcf9d: Waiting for close lock at 1732018413285Running coprocessor pre-close hooks at 1732018413285Disabling compacts and flushes for region at 1732018413285Disabling writes for close at 1732018413285Writing region close event to WAL at 1732018413286 (+1 ms)Running coprocessor post-close hooks at 1732018413293 (+7 ms)Closed at 1732018413293 2024-11-19T12:13:33,294 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1732018406961.8fbade10b0e1d2be0489df2d6c3fcf9d. 2024-11-19T12:13:33,294 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 24b360a2b049dbc9c784cdd687321115, disabling compactions & flushes 2024-11-19T12:13:33,294 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/2b6d3f4b598ab015d9c62b67f0bac50f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,294 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115. 2024-11-19T12:13:33,294 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115. 2024-11-19T12:13:33,294 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115. after waiting 0 ms 2024-11-19T12:13:33,294 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115. 2024-11-19T12:13:33,295 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1732018406961.2b6d3f4b598ab015d9c62b67f0bac50f. 2024-11-19T12:13:33,295 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2b6d3f4b598ab015d9c62b67f0bac50f: Waiting for close lock at 1732018413285Running coprocessor pre-close hooks at 1732018413285Disabling compacts and flushes for region at 1732018413285Disabling writes for close at 1732018413285Writing region close event to WAL at 1732018413288 (+3 ms)Running coprocessor post-close hooks at 1732018413294 (+6 ms)Closed at 1732018413294 2024-11-19T12:13:33,295 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1732018406961.2b6d3f4b598ab015d9c62b67f0bac50f. 2024-11-19T12:13:33,295 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing af2a9e847c9e4d4812af432c436c7ac0, disabling compactions & flushes 2024-11-19T12:13:33,295 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1732018406961.af2a9e847c9e4d4812af432c436c7ac0. 2024-11-19T12:13:33,295 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732018406961.af2a9e847c9e4d4812af432c436c7ac0. 2024-11-19T12:13:33,295 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732018406961.af2a9e847c9e4d4812af432c436c7ac0. after waiting 0 ms 2024-11-19T12:13:33,295 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732018406961.af2a9e847c9e4d4812af432c436c7ac0. 2024-11-19T12:13:33,307 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/24b360a2b049dbc9c784cdd687321115/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,308 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115. 2024-11-19T12:13:33,308 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 24b360a2b049dbc9c784cdd687321115: Waiting for close lock at 1732018413294Running coprocessor pre-close hooks at 1732018413294Disabling compacts and flushes for region at 1732018413294Disabling writes for close at 1732018413294Writing region close event to WAL at 1732018413300 (+6 ms)Running coprocessor post-close hooks at 1732018413308 (+8 ms)Closed at 1732018413308 2024-11-19T12:13:33,308 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1732018406961.24b360a2b049dbc9c784cdd687321115. 2024-11-19T12:13:33,309 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5bc8ba883b6bd714e90daec2cb61209e, disabling compactions & flushes 2024-11-19T12:13:33,309 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e. 2024-11-19T12:13:33,309 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e. 2024-11-19T12:13:33,309 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e. after waiting 0 ms 2024-11-19T12:13:33,309 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e. 2024-11-19T12:13:33,309 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/af2a9e847c9e4d4812af432c436c7ac0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741869_1045 (size=8177) 2024-11-19T12:13:33,310 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1732018406961.af2a9e847c9e4d4812af432c436c7ac0. 2024-11-19T12:13:33,310 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for af2a9e847c9e4d4812af432c436c7ac0: Waiting for close lock at 1732018413295Running coprocessor pre-close hooks at 1732018413295Disabling compacts and flushes for region at 1732018413295Disabling writes for close at 1732018413295Writing region close event to WAL at 1732018413301 (+6 ms)Running coprocessor post-close hooks at 1732018413310 (+9 ms)Closed at 1732018413310 2024-11-19T12:13:33,310 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1732018406961.af2a9e847c9e4d4812af432c436c7ac0. 2024-11-19T12:13:33,310 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 19add541d34392b48fdd488f1c19c182, disabling compactions & flushes 2024-11-19T12:13:33,310 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1732018406961.19add541d34392b48fdd488f1c19c182. 2024-11-19T12:13:33,310 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732018406961.19add541d34392b48fdd488f1c19c182. 2024-11-19T12:13:33,311 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732018406961.19add541d34392b48fdd488f1c19c182. after waiting 0 ms 2024-11-19T12:13:33,311 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732018406961.19add541d34392b48fdd488f1c19c182. 2024-11-19T12:13:33,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741869_1045 (size=8177) 2024-11-19T12:13:33,311 INFO [RS_CLOSE_META-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.64 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/.tmp/info/033f3aaa4cee40e6b704061229c5002c 2024-11-19T12:13:33,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741869_1045 (size=8177) 2024-11-19T12:13:33,314 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/5bc8ba883b6bd714e90daec2cb61209e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,315 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e. 2024-11-19T12:13:33,316 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5bc8ba883b6bd714e90daec2cb61209e: Waiting for close lock at 1732018413308Running coprocessor pre-close hooks at 1732018413308Disabling compacts and flushes for region at 1732018413309 (+1 ms)Disabling writes for close at 1732018413309Writing region close event to WAL at 1732018413310 (+1 ms)Running coprocessor post-close hooks at 1732018413315 (+5 ms)Closed at 1732018413315 2024-11-19T12:13:33,316 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/19add541d34392b48fdd488f1c19c182/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,316 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1732018406961.5bc8ba883b6bd714e90daec2cb61209e. 2024-11-19T12:13:33,316 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ca6b3c7ec060f344fff683e91e796d33, disabling compactions & flushes 2024-11-19T12:13:33,316 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1732018406961.ca6b3c7ec060f344fff683e91e796d33. 2024-11-19T12:13:33,316 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732018406961.ca6b3c7ec060f344fff683e91e796d33. 2024-11-19T12:13:33,316 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732018406961.ca6b3c7ec060f344fff683e91e796d33. after waiting 0 ms 2024-11-19T12:13:33,316 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732018406961.ca6b3c7ec060f344fff683e91e796d33. 2024-11-19T12:13:33,317 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1732018406961.19add541d34392b48fdd488f1c19c182. 2024-11-19T12:13:33,318 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 19add541d34392b48fdd488f1c19c182: Waiting for close lock at 1732018413310Running coprocessor pre-close hooks at 1732018413310Disabling compacts and flushes for region at 1732018413310Disabling writes for close at 1732018413311 (+1 ms)Writing region close event to WAL at 1732018413311Running coprocessor post-close hooks at 1732018413317 (+6 ms)Closed at 1732018413317 2024-11-19T12:13:33,318 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1732018406961.19add541d34392b48fdd488f1c19c182. 2024-11-19T12:13:33,318 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing adc3abaf30d887a0d57a7730313a62b4, disabling compactions & flushes 2024-11-19T12:13:33,318 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1732018406961.adc3abaf30d887a0d57a7730313a62b4. 2024-11-19T12:13:33,318 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732018406961.adc3abaf30d887a0d57a7730313a62b4. 2024-11-19T12:13:33,318 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732018406961.adc3abaf30d887a0d57a7730313a62b4. after waiting 0 ms 2024-11-19T12:13:33,318 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732018406961.adc3abaf30d887a0d57a7730313a62b4. 2024-11-19T12:13:33,322 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/ca6b3c7ec060f344fff683e91e796d33/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,322 DEBUG [RS_CLOSE_META-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/.tmp/info/033f3aaa4cee40e6b704061229c5002c as hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/info/033f3aaa4cee40e6b704061229c5002c 2024-11-19T12:13:33,322 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1732018406961.ca6b3c7ec060f344fff683e91e796d33. 2024-11-19T12:13:33,323 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ca6b3c7ec060f344fff683e91e796d33: Waiting for close lock at 1732018413316Running coprocessor pre-close hooks at 1732018413316Disabling compacts and flushes for region at 1732018413316Disabling writes for close at 1732018413316Writing region close event to WAL at 1732018413317 (+1 ms)Running coprocessor post-close hooks at 1732018413322 (+5 ms)Closed at 1732018413322 2024-11-19T12:13:33,323 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1732018406961.ca6b3c7ec060f344fff683e91e796d33. 2024-11-19T12:13:33,323 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4fea5432239dac4bfb1b607057a95e34, disabling compactions & flushes 2024-11-19T12:13:33,323 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1732018406961.4fea5432239dac4bfb1b607057a95e34. 2024-11-19T12:13:33,323 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732018406961.4fea5432239dac4bfb1b607057a95e34. 2024-11-19T12:13:33,323 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732018406961.4fea5432239dac4bfb1b607057a95e34. after waiting 0 ms 2024-11-19T12:13:33,323 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732018406961.4fea5432239dac4bfb1b607057a95e34. 2024-11-19T12:13:33,325 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/adc3abaf30d887a0d57a7730313a62b4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,326 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1732018406961.adc3abaf30d887a0d57a7730313a62b4. 2024-11-19T12:13:33,326 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for adc3abaf30d887a0d57a7730313a62b4: Waiting for close lock at 1732018413318Running coprocessor pre-close hooks at 1732018413318Disabling compacts and flushes for region at 1732018413318Disabling writes for close at 1732018413318Writing region close event to WAL at 1732018413319 (+1 ms)Running coprocessor post-close hooks at 1732018413326 (+7 ms)Closed at 1732018413326 2024-11-19T12:13:33,327 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1732018406961.adc3abaf30d887a0d57a7730313a62b4. 2024-11-19T12:13:33,327 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b859464d4ffda6f2df23654fb949364f, disabling compactions & flushes 2024-11-19T12:13:33,327 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1732018406961.b859464d4ffda6f2df23654fb949364f. 2024-11-19T12:13:33,327 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732018406961.b859464d4ffda6f2df23654fb949364f. 2024-11-19T12:13:33,327 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1732018406961.b859464d4ffda6f2df23654fb949364f. after waiting 0 ms 2024-11-19T12:13:33,327 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1732018406961.b859464d4ffda6f2df23654fb949364f. 2024-11-19T12:13:33,330 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/4fea5432239dac4bfb1b607057a95e34/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,331 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1732018406961.4fea5432239dac4bfb1b607057a95e34. 2024-11-19T12:13:33,331 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4fea5432239dac4bfb1b607057a95e34: Waiting for close lock at 1732018413323Running coprocessor pre-close hooks at 1732018413323Disabling compacts and flushes for region at 1732018413323Disabling writes for close at 1732018413323Writing region close event to WAL at 1732018413324 (+1 ms)Running coprocessor post-close hooks at 1732018413331 (+7 ms)Closed at 1732018413331 2024-11-19T12:13:33,332 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1732018406961.4fea5432239dac4bfb1b607057a95e34. 2024-11-19T12:13:33,332 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f9224ed53fa7acfead92d4143c8626ff, disabling compactions & flushes 2024-11-19T12:13:33,332 INFO [RS_CLOSE_META-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/info/033f3aaa4cee40e6b704061229c5002c, entries=24, sequenceid=75, filesize=8.0 K 2024-11-19T12:13:33,332 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff. 2024-11-19T12:13:33,332 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff. 2024-11-19T12:13:33,332 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff. after waiting 0 ms 2024-11-19T12:13:33,332 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff. 2024-11-19T12:13:33,333 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/b859464d4ffda6f2df23654fb949364f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,334 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1732018406961.b859464d4ffda6f2df23654fb949364f. 2024-11-19T12:13:33,334 INFO [RS_CLOSE_META-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 61ms, sequenceid=75, compaction requested=false 2024-11-19T12:13:33,334 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b859464d4ffda6f2df23654fb949364f: Waiting for close lock at 1732018413327Running coprocessor pre-close hooks at 1732018413327Disabling compacts and flushes for region at 1732018413327Disabling writes for close at 1732018413327Writing region close event to WAL at 1732018413327Running coprocessor post-close hooks at 1732018413333 (+6 ms)Closed at 1732018413333 2024-11-19T12:13:33,334 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1732018406961.b859464d4ffda6f2df23654fb949364f. 2024-11-19T12:13:33,334 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a78d1de57ec917279b938f869bdbe10d, disabling compactions & flushes 2024-11-19T12:13:33,334 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1732018406961.a78d1de57ec917279b938f869bdbe10d. 2024-11-19T12:13:33,334 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732018406961.a78d1de57ec917279b938f869bdbe10d. 2024-11-19T12:13:33,334 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732018406961.a78d1de57ec917279b938f869bdbe10d. after waiting 0 ms 2024-11-19T12:13:33,334 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732018406961.a78d1de57ec917279b938f869bdbe10d. 2024-11-19T12:13:33,341 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/f9224ed53fa7acfead92d4143c8626ff/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,342 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff. 2024-11-19T12:13:33,342 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f9224ed53fa7acfead92d4143c8626ff: Waiting for close lock at 1732018413332Running coprocessor pre-close hooks at 1732018413332Disabling compacts and flushes for region at 1732018413332Disabling writes for close at 1732018413332Writing region close event to WAL at 1732018413333 (+1 ms)Running coprocessor post-close hooks at 1732018413342 (+9 ms)Closed at 1732018413342 2024-11-19T12:13:33,342 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1732018406961.f9224ed53fa7acfead92d4143c8626ff. 2024-11-19T12:13:33,343 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e85bee36dde7095f8d08f860a32613d7, disabling compactions & flushes 2024-11-19T12:13:33,343 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1732018406961.e85bee36dde7095f8d08f860a32613d7. 2024-11-19T12:13:33,343 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732018406961.e85bee36dde7095f8d08f860a32613d7. 2024-11-19T12:13:33,343 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1732018406961.e85bee36dde7095f8d08f860a32613d7. after waiting 0 ms 2024-11-19T12:13:33,343 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1732018406961.e85bee36dde7095f8d08f860a32613d7. 2024-11-19T12:13:33,345 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/a78d1de57ec917279b938f869bdbe10d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,345 DEBUG [RS_CLOSE_META-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-11-19T12:13:33,345 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1732018406961.a78d1de57ec917279b938f869bdbe10d. 2024-11-19T12:13:33,345 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a78d1de57ec917279b938f869bdbe10d: Waiting for close lock at 1732018413334Running coprocessor pre-close hooks at 1732018413334Disabling compacts and flushes for region at 1732018413334Disabling writes for close at 1732018413334Writing region close event to WAL at 1732018413340 (+6 ms)Running coprocessor post-close hooks at 1732018413345 (+5 ms)Closed at 1732018413345 2024-11-19T12:13:33,346 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1732018406961.a78d1de57ec917279b938f869bdbe10d. 2024-11-19T12:13:33,346 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 963c42e1179165051d86c1560642c124, disabling compactions & flushes 2024-11-19T12:13:33,346 DEBUG [RS_CLOSE_META-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T12:13:33,346 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1732018406961.963c42e1179165051d86c1560642c124. 2024-11-19T12:13:33,346 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732018406961.963c42e1179165051d86c1560642c124. 2024-11-19T12:13:33,346 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732018406961.963c42e1179165051d86c1560642c124. after waiting 0 ms 2024-11-19T12:13:33,346 INFO [RS_CLOSE_META-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T12:13:33,346 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732018406961.963c42e1179165051d86c1560642c124. 2024-11-19T12:13:33,346 DEBUG [RS_CLOSE_META-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732018413272Running coprocessor pre-close hooks at 1732018413272Disabling compacts and flushes for region at 1732018413272Disabling writes for close at 1732018413272Obtaining lock to block concurrent updates at 1732018413273 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732018413273Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2708, getHeapSize=5488, getOffHeapSize=0, getCellsCount=24 at 1732018413279 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732018413281 (+2 ms)Flushing 1588230740/info: creating writer at 1732018413281Flushing 1588230740/info: appending metadata at 1732018413287 (+6 ms)Flushing 1588230740/info: closing flushed file at 1732018413287Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b38d6c8: reopening flushed file at 1732018413321 (+34 ms)Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 61ms, sequenceid=75, compaction requested=false at 1732018413334 (+13 ms)Writing region close event to WAL at 1732018413342 (+8 ms)Running coprocessor post-close hooks at 1732018413346 (+4 ms)Closed at 1732018413346 2024-11-19T12:13:33,346 DEBUG [RS_CLOSE_META-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T12:13:33,347 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/e85bee36dde7095f8d08f860a32613d7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,348 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1732018406961.e85bee36dde7095f8d08f860a32613d7. 2024-11-19T12:13:33,348 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e85bee36dde7095f8d08f860a32613d7: Waiting for close lock at 1732018413343Running coprocessor pre-close hooks at 1732018413343Disabling compacts and flushes for region at 1732018413343Disabling writes for close at 1732018413343Writing region close event to WAL at 1732018413343Running coprocessor post-close hooks at 1732018413348 (+5 ms)Closed at 1732018413348 2024-11-19T12:13:33,349 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1732018406961.e85bee36dde7095f8d08f860a32613d7. 2024-11-19T12:13:33,349 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2ec97ffd6b2cc998be397eec7fc0a0bc, disabling compactions & flushes 2024-11-19T12:13:33,349 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1732018406961.2ec97ffd6b2cc998be397eec7fc0a0bc. 2024-11-19T12:13:33,349 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732018406961.2ec97ffd6b2cc998be397eec7fc0a0bc. 2024-11-19T12:13:33,349 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732018406961.2ec97ffd6b2cc998be397eec7fc0a0bc. after waiting 0 ms 2024-11-19T12:13:33,349 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732018406961.2ec97ffd6b2cc998be397eec7fc0a0bc. 2024-11-19T12:13:33,351 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/963c42e1179165051d86c1560642c124/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,351 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1732018406961.963c42e1179165051d86c1560642c124. 2024-11-19T12:13:33,351 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 963c42e1179165051d86c1560642c124: Waiting for close lock at 1732018413346Running coprocessor pre-close hooks at 1732018413346Disabling compacts and flushes for region at 1732018413346Disabling writes for close at 1732018413346Writing region close event to WAL at 1732018413346Running coprocessor post-close hooks at 1732018413351 (+5 ms)Closed at 1732018413351 2024-11-19T12:13:33,352 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1732018406961.963c42e1179165051d86c1560642c124. 2024-11-19T12:13:33,352 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d8bfccd6e991fb7a90ce5bcadc0a3ef5, disabling compactions & flushes 2024-11-19T12:13:33,352 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5. 2024-11-19T12:13:33,352 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5. 2024-11-19T12:13:33,352 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5. after waiting 0 ms 2024-11-19T12:13:33,352 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5. 2024-11-19T12:13:33,353 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/2ec97ffd6b2cc998be397eec7fc0a0bc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,354 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1732018406961.2ec97ffd6b2cc998be397eec7fc0a0bc. 2024-11-19T12:13:33,354 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2ec97ffd6b2cc998be397eec7fc0a0bc: Waiting for close lock at 1732018413349Running coprocessor pre-close hooks at 1732018413349Disabling compacts and flushes for region at 1732018413349Disabling writes for close at 1732018413349Writing region close event to WAL at 1732018413350 (+1 ms)Running coprocessor post-close hooks at 1732018413354 (+4 ms)Closed at 1732018413354 2024-11-19T12:13:33,354 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1732018406961.2ec97ffd6b2cc998be397eec7fc0a0bc. 2024-11-19T12:13:33,354 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ef4298fd6621c8e7facf42422bd80be4, disabling compactions & flushes 2024-11-19T12:13:33,354 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1732018406961.ef4298fd6621c8e7facf42422bd80be4. 2024-11-19T12:13:33,355 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732018406961.ef4298fd6621c8e7facf42422bd80be4. 2024-11-19T12:13:33,355 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732018406961.ef4298fd6621c8e7facf42422bd80be4. after waiting 0 ms 2024-11-19T12:13:33,355 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732018406961.ef4298fd6621c8e7facf42422bd80be4. 2024-11-19T12:13:33,356 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/d8bfccd6e991fb7a90ce5bcadc0a3ef5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,357 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5. 2024-11-19T12:13:33,357 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d8bfccd6e991fb7a90ce5bcadc0a3ef5: Waiting for close lock at 1732018413352Running coprocessor pre-close hooks at 1732018413352Disabling compacts and flushes for region at 1732018413352Disabling writes for close at 1732018413352Writing region close event to WAL at 1732018413352Running coprocessor post-close hooks at 1732018413357 (+5 ms)Closed at 1732018413357 2024-11-19T12:13:33,357 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1732018406961.d8bfccd6e991fb7a90ce5bcadc0a3ef5. 2024-11-19T12:13:33,357 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e257775d8750f12b70161bb0b6387163, disabling compactions & flushes 2024-11-19T12:13:33,357 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1732018406961.e257775d8750f12b70161bb0b6387163. 2024-11-19T12:13:33,357 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732018406961.e257775d8750f12b70161bb0b6387163. 2024-11-19T12:13:33,357 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732018406961.e257775d8750f12b70161bb0b6387163. after waiting 0 ms 2024-11-19T12:13:33,358 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732018406961.e257775d8750f12b70161bb0b6387163. 2024-11-19T12:13:33,359 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/ef4298fd6621c8e7facf42422bd80be4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,360 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1732018406961.ef4298fd6621c8e7facf42422bd80be4. 2024-11-19T12:13:33,360 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ef4298fd6621c8e7facf42422bd80be4: Waiting for close lock at 1732018413354Running coprocessor pre-close hooks at 1732018413354Disabling compacts and flushes for region at 1732018413354Disabling writes for close at 1732018413355 (+1 ms)Writing region close event to WAL at 1732018413355Running coprocessor post-close hooks at 1732018413360 (+5 ms)Closed at 1732018413360 2024-11-19T12:13:33,360 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1732018406961.ef4298fd6621c8e7facf42422bd80be4. 2024-11-19T12:13:33,360 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5b9b9ba699dcd89ef4b3398f2beb2455, disabling compactions & flushes 2024-11-19T12:13:33,361 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1732018406961.5b9b9ba699dcd89ef4b3398f2beb2455. 2024-11-19T12:13:33,361 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732018406961.5b9b9ba699dcd89ef4b3398f2beb2455. 2024-11-19T12:13:33,361 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732018406961.5b9b9ba699dcd89ef4b3398f2beb2455. after waiting 0 ms 2024-11-19T12:13:33,361 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732018406961.5b9b9ba699dcd89ef4b3398f2beb2455. 2024-11-19T12:13:33,362 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/e257775d8750f12b70161bb0b6387163/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,362 INFO [regionserver/351a98347401:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T12:13:33,363 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1732018406961.e257775d8750f12b70161bb0b6387163. 2024-11-19T12:13:33,363 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e257775d8750f12b70161bb0b6387163: Waiting for close lock at 1732018413357Running coprocessor pre-close hooks at 1732018413357Disabling compacts and flushes for region at 1732018413357Disabling writes for close at 1732018413358 (+1 ms)Writing region close event to WAL at 1732018413358Running coprocessor post-close hooks at 1732018413363 (+5 ms)Closed at 1732018413363 2024-11-19T12:13:33,363 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1732018406961.e257775d8750f12b70161bb0b6387163. 2024-11-19T12:13:33,363 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c48a9e94bd9fcbd6f8c1f6ae2b6c80d3, disabling compactions & flushes 2024-11-19T12:13:33,363 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3. 2024-11-19T12:13:33,363 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3. 2024-11-19T12:13:33,363 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3. after waiting 0 ms 2024-11-19T12:13:33,363 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3. 2024-11-19T12:13:33,364 INFO [regionserver/351a98347401:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T12:13:33,365 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/5b9b9ba699dcd89ef4b3398f2beb2455/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,366 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1732018406961.5b9b9ba699dcd89ef4b3398f2beb2455. 2024-11-19T12:13:33,366 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5b9b9ba699dcd89ef4b3398f2beb2455: Waiting for close lock at 1732018413360Running coprocessor pre-close hooks at 1732018413360Disabling compacts and flushes for region at 1732018413360Disabling writes for close at 1732018413361 (+1 ms)Writing region close event to WAL at 1732018413361Running coprocessor post-close hooks at 1732018413366 (+5 ms)Closed at 1732018413366 2024-11-19T12:13:33,366 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1732018406961.5b9b9ba699dcd89ef4b3398f2beb2455. 2024-11-19T12:13:33,366 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 11d894b29c3ee7cc1e5281c570908571, disabling compactions & flushes 2024-11-19T12:13:33,366 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571. 2024-11-19T12:13:33,366 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571. 2024-11-19T12:13:33,366 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571. after waiting 0 ms 2024-11-19T12:13:33,366 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571. 2024-11-19T12:13:33,368 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/c48a9e94bd9fcbd6f8c1f6ae2b6c80d3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,369 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3. 2024-11-19T12:13:33,369 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c48a9e94bd9fcbd6f8c1f6ae2b6c80d3: Waiting for close lock at 1732018413363Running coprocessor pre-close hooks at 1732018413363Disabling compacts and flushes for region at 1732018413363Disabling writes for close at 1732018413363Writing region close event to WAL at 1732018413364 (+1 ms)Running coprocessor post-close hooks at 1732018413369 (+5 ms)Closed at 1732018413369 2024-11-19T12:13:33,369 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1732018406961.c48a9e94bd9fcbd6f8c1f6ae2b6c80d3. 2024-11-19T12:13:33,369 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7382ad40044282a358da8130ce023367, disabling compactions & flushes 2024-11-19T12:13:33,369 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1732018406961.7382ad40044282a358da8130ce023367. 2024-11-19T12:13:33,369 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732018406961.7382ad40044282a358da8130ce023367. 2024-11-19T12:13:33,369 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732018406961.7382ad40044282a358da8130ce023367. after waiting 0 ms 2024-11-19T12:13:33,369 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732018406961.7382ad40044282a358da8130ce023367. 2024-11-19T12:13:33,371 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/11d894b29c3ee7cc1e5281c570908571/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T12:13:33,371 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571. 2024-11-19T12:13:33,371 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 11d894b29c3ee7cc1e5281c570908571: Waiting for close lock at 1732018413366Running coprocessor pre-close hooks at 1732018413366Disabling compacts and flushes for region at 1732018413366Disabling writes for close at 1732018413366Writing region close event to WAL at 1732018413367 (+1 ms)Running coprocessor post-close hooks at 1732018413371 (+4 ms)Closed at 1732018413371 2024-11-19T12:13:33,372 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1732018406961.11d894b29c3ee7cc1e5281c570908571. 2024-11-19T12:13:33,373 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/data/default/test/7382ad40044282a358da8130ce023367/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-19T12:13:33,374 INFO [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1732018406961.7382ad40044282a358da8130ce023367. 2024-11-19T12:13:33,374 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7382ad40044282a358da8130ce023367: Waiting for close lock at 1732018413369Running coprocessor pre-close hooks at 1732018413369Disabling compacts and flushes for region at 1732018413369Disabling writes for close at 1732018413369Writing region close event to WAL at 1732018413370 (+1 ms)Running coprocessor post-close hooks at 1732018413374 (+4 ms)Closed at 1732018413374 2024-11-19T12:13:33,374 DEBUG [RS_CLOSE_REGION-regionserver/351a98347401:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1732018406961.7382ad40044282a358da8130ce023367. 2024-11-19T12:13:33,471 INFO [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(976): stopping server 351a98347401,45367,1732018404555; all regions closed. 2024-11-19T12:13:33,472 INFO [RS:0;351a98347401:36275 {}] regionserver.HRegionServer(976): stopping server 351a98347401,36275,1732018404438; all regions closed. 2024-11-19T12:13:33,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741865_1041 (size=4192) 2024-11-19T12:13:33,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741835_1011 (size=7074) 2024-11-19T12:13:33,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741865_1041 (size=4192) 2024-11-19T12:13:33,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741865_1041 (size=4192) 2024-11-19T12:13:33,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741835_1011 (size=7074) 2024-11-19T12:13:33,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741835_1011 (size=7074) 2024-11-19T12:13:33,482 DEBUG [RS:0;351a98347401:36275 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/oldWALs 2024-11-19T12:13:33,482 INFO [RS:0;351a98347401:36275 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 351a98347401%2C36275%2C1732018404438.meta:.meta(num 1732018411212) 2024-11-19T12:13:33,483 DEBUG [RS:1;351a98347401:45367 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/oldWALs 2024-11-19T12:13:33,483 INFO [RS:1;351a98347401:45367 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 351a98347401%2C45367%2C1732018404555:(num 1732018406120) 2024-11-19T12:13:33,483 DEBUG [RS:1;351a98347401:45367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:13:33,483 INFO [RS:1;351a98347401:45367 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T12:13:33,483 INFO [RS:1;351a98347401:45367 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T12:13:33,483 INFO [RS:1;351a98347401:45367 {}] hbase.ChoreService(370): Chore service for: regionserver/351a98347401:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T12:13:33,483 INFO [RS:1;351a98347401:45367 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T12:13:33,483 INFO [RS:1;351a98347401:45367 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T12:13:33,483 INFO [RS:1;351a98347401:45367 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T12:13:33,483 INFO [regionserver/351a98347401:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T12:13:33,483 INFO [RS:1;351a98347401:45367 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T12:13:33,484 INFO [RS:1;351a98347401:45367 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45367 2024-11-19T12:13:33,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741834_1010 (size=7064) 2024-11-19T12:13:33,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741834_1010 (size=7064) 2024-11-19T12:13:33,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741834_1010 (size=7064) 2024-11-19T12:13:33,487 DEBUG [RS:0;351a98347401:36275 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/oldWALs 2024-11-19T12:13:33,487 INFO [RS:0;351a98347401:36275 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 351a98347401%2C36275%2C1732018404438:(num 1732018406120) 2024-11-19T12:13:33,487 DEBUG [RS:0;351a98347401:36275 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:13:33,487 INFO [RS:0;351a98347401:36275 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T12:13:33,487 INFO [RS:0;351a98347401:36275 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T12:13:33,488 INFO [RS:0;351a98347401:36275 {}] hbase.ChoreService(370): Chore service for: regionserver/351a98347401:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T12:13:33,488 INFO [RS:0;351a98347401:36275 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T12:13:33,488 INFO [regionserver/351a98347401:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T12:13:33,488 INFO [RS:0;351a98347401:36275 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36275 2024-11-19T12:13:33,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T12:13:33,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/351a98347401,45367,1732018404555 2024-11-19T12:13:33,489 INFO [RS:1;351a98347401:45367 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T12:13:33,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36275-0x1015303718b0001, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/351a98347401,36275,1732018404438 2024-11-19T12:13:33,500 INFO [RS:0;351a98347401:36275 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T12:13:33,510 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [351a98347401,45367,1732018404555] 2024-11-19T12:13:33,532 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/351a98347401,45367,1732018404555 already deleted, retry=false 2024-11-19T12:13:33,532 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 351a98347401,45367,1732018404555 expired; onlineServers=1 2024-11-19T12:13:33,532 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [351a98347401,36275,1732018404438] 2024-11-19T12:13:33,542 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/351a98347401,36275,1732018404438 already deleted, retry=false 2024-11-19T12:13:33,542 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 351a98347401,36275,1732018404438 expired; onlineServers=0 2024-11-19T12:13:33,543 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '351a98347401,38839,1732018403620' ***** 2024-11-19T12:13:33,543 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-19T12:13:33,543 INFO [M:0;351a98347401:38839 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T12:13:33,544 INFO [M:0;351a98347401:38839 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T12:13:33,544 DEBUG [M:0;351a98347401:38839 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T12:13:33,544 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T12:13:33,544 DEBUG [M:0;351a98347401:38839 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T12:13:33,544 DEBUG [master/351a98347401:0:becomeActiveMaster-HFileCleaner.small.0-1732018405723 {}] cleaner.HFileCleaner(306): Exit Thread[master/351a98347401:0:becomeActiveMaster-HFileCleaner.small.0-1732018405723,5,FailOnTimeoutGroup] 2024-11-19T12:13:33,544 DEBUG [master/351a98347401:0:becomeActiveMaster-HFileCleaner.large.0-1732018405722 {}] cleaner.HFileCleaner(306): Exit Thread[master/351a98347401:0:becomeActiveMaster-HFileCleaner.large.0-1732018405722,5,FailOnTimeoutGroup] 2024-11-19T12:13:33,545 INFO [M:0;351a98347401:38839 {}] hbase.ChoreService(370): Chore service for: master/351a98347401:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T12:13:33,545 INFO [M:0;351a98347401:38839 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T12:13:33,546 DEBUG [M:0;351a98347401:38839 {}] master.HMaster(1795): Stopping service threads 2024-11-19T12:13:33,546 INFO [M:0;351a98347401:38839 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T12:13:33,546 INFO [M:0;351a98347401:38839 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:13:33,547 INFO [M:0;351a98347401:38839 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:13:33,547 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T12:13:33,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-19T12:13:33,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:13:33,553 DEBUG [M:0;351a98347401:38839 {}] zookeeper.ZKUtil(347): master:38839-0x1015303718b0000, quorum=127.0.0.1:57377, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-19T12:13:33,553 WARN [M:0;351a98347401:38839 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T12:13:33,553 INFO [M:0;351a98347401:38839 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/.lastflushedseqids 2024-11-19T12:13:33,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741870_1046 (size=1) 2024-11-19T12:13:33,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741870_1046 (size=1) 2024-11-19T12:13:33,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741870_1046 (size=1) 2024-11-19T12:13:33,565 INFO [M:0;351a98347401:38839 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:13:33,565 INFO [M:0;351a98347401:38839 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T12:13:33,566 DEBUG [M:0;351a98347401:38839 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T12:13:33,566 INFO [M:0;351a98347401:38839 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:13:33,566 DEBUG [M:0;351a98347401:38839 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:13:33,566 DEBUG [M:0;351a98347401:38839 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T12:13:33,566 DEBUG [M:0;351a98347401:38839 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:13:33,566 INFO [M:0;351a98347401:38839 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=260.79 KB heapSize=317.02 KB 2024-11-19T12:13:33,581 DEBUG [M:0;351a98347401:38839 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e17c775448d44fd9a9e2ba8d42965bb2 is 82, key is hbase:meta,,1/info:regioninfo/1732018411538/Put/seqid=0 2024-11-19T12:13:33,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46231 is added to blk_1073741871_1047 (size=6071) 2024-11-19T12:13:33,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40579 is added to blk_1073741871_1047 (size=6071) 2024-11-19T12:13:33,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46857 is added to blk_1073741871_1047 (size=6071) 2024-11-19T12:13:33,590 INFO [M:0;351a98347401:38839 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=603 (bloomFilter=true), to=hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e17c775448d44fd9a9e2ba8d42965bb2 2024-11-19T12:13:33,611 INFO [RS:1;351a98347401:45367 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T12:13:33,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T12:13:33,611 INFO [RS:1;351a98347401:45367 {}] regionserver.HRegionServer(1031): Exiting; stopping=351a98347401,45367,1732018404555; zookeeper connection closed. 2024-11-19T12:13:33,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45367-0x1015303718b0002, quorum=127.0.0.1:57377, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T12:13:33,611 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@27638bf0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@27638bf0 2024-11-19T12:13:33,613 DEBUG [M:0;351a98347401:38839 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36199/user/jenkins/test-data/4210f08e-1632-b3a2-52c7-aa305039bbc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9449acc6ce3849169a097b4e91eddb78 is 2987, key is \x00\x00\x00\x00\x00\x00\x00