2024-11-09 05:40:13,703 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-09 05:40:13,727 main DEBUG Took 0.021271 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-09 05:40:13,727 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-09 05:40:13,728 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-09 05:40:13,729 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-09 05:40:13,731 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 05:40:13,740 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-09 05:40:13,756 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 05:40:13,758 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 05:40:13,759 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 05:40:13,759 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 05:40:13,760 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 05:40:13,760 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 05:40:13,762 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 05:40:13,762 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 05:40:13,762 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 05:40:13,763 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 05:40:13,764 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 05:40:13,764 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 05:40:13,765 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 05:40:13,765 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 05:40:13,766 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 05:40:13,766 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 05:40:13,767 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 05:40:13,767 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 05:40:13,768 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 05:40:13,769 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 05:40:13,769 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 05:40:13,770 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 05:40:13,770 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 05:40:13,771 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 05:40:13,771 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 05:40:13,772 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-09 05:40:13,774 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 05:40:13,775 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-09 05:40:13,777 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-09 05:40:13,778 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-09 05:40:13,779 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-09 05:40:13,780 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-09 05:40:13,791 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-09 05:40:13,794 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-09 05:40:13,796 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-09 05:40:13,796 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-09 05:40:13,797 main DEBUG createAppenders(={Console}) 2024-11-09 05:40:13,799 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-09 05:40:13,799 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-09 05:40:13,800 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-09 05:40:13,801 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-09 05:40:13,801 main DEBUG OutputStream closed 2024-11-09 05:40:13,801 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-09 05:40:13,802 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-09 05:40:13,802 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-09 05:40:13,929 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-09 05:40:13,932 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-09 05:40:13,934 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-09 05:40:13,935 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-09 05:40:13,936 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-09 05:40:13,936 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-09 05:40:13,937 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-09 05:40:13,937 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-09 05:40:13,938 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-09 05:40:13,938 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-09 05:40:13,939 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-09 05:40:13,939 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-09 05:40:13,939 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-09 05:40:13,940 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-09 05:40:13,940 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-09 05:40:13,941 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-09 05:40:13,941 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-09 05:40:13,942 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-09 05:40:13,945 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09 05:40:13,946 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-09 05:40:13,946 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-09 05:40:13,947 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-09T05:40:13,965 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-09 05:40:13,973 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-09 05:40:13,974 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09T05:40:14,375 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82 2024-11-09T05:40:14,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-09T05:40:14,424 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/cluster_a63c0870-ce87-d57c-f3a1-ecd7f1f7e9af, deleteOnExit=true 2024-11-09T05:40:14,424 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-09T05:40:14,425 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/test.cache.data in system properties and HBase conf 2024-11-09T05:40:14,426 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/hadoop.tmp.dir in system properties and HBase conf 2024-11-09T05:40:14,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/hadoop.log.dir in system properties and HBase conf 2024-11-09T05:40:14,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-09T05:40:14,428 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-09T05:40:14,428 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-09T05:40:14,550 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-09T05:40:14,681 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T05:40:14,686 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-09T05:40:14,687 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-09T05:40:14,687 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-09T05:40:14,688 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T05:40:14,689 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-09T05:40:14,689 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-09T05:40:14,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T05:40:14,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T05:40:14,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-09T05:40:14,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/nfs.dump.dir in system properties and HBase conf 2024-11-09T05:40:14,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/java.io.tmpdir in system properties and HBase conf 2024-11-09T05:40:14,693 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T05:40:14,693 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-09T05:40:14,694 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-09T05:40:15,669 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-09T05:40:15,761 INFO [Time-limited test {}] log.Log(170): Logging initialized @2943ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-09T05:40:15,862 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T05:40:15,950 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T05:40:15,976 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T05:40:15,977 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T05:40:15,978 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-09T05:40:15,992 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T05:40:15,995 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/hadoop.log.dir/,AVAILABLE} 2024-11-09T05:40:15,996 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T05:40:16,213 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/java.io.tmpdir/jetty-localhost-41531-hadoop-hdfs-3_4_1-tests_jar-_-any-7198421189597048869/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T05:40:16,228 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:41531} 2024-11-09T05:40:16,228 INFO [Time-limited test {}] server.Server(415): Started @3411ms 2024-11-09T05:40:16,687 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T05:40:16,695 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T05:40:16,697 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T05:40:16,697 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T05:40:16,697 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-09T05:40:16,698 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/hadoop.log.dir/,AVAILABLE} 2024-11-09T05:40:16,699 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T05:40:16,830 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/java.io.tmpdir/jetty-localhost-45427-hadoop-hdfs-3_4_1-tests_jar-_-any-17238465202779539057/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T05:40:16,831 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:45427} 2024-11-09T05:40:16,831 INFO [Time-limited test {}] server.Server(415): Started @4014ms 2024-11-09T05:40:16,890 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-09T05:40:17,068 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T05:40:17,086 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T05:40:17,107 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T05:40:17,108 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T05:40:17,108 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-09T05:40:17,110 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/hadoop.log.dir/,AVAILABLE} 2024-11-09T05:40:17,111 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T05:40:17,278 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/java.io.tmpdir/jetty-localhost-46843-hadoop-hdfs-3_4_1-tests_jar-_-any-8416286742949775578/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T05:40:17,279 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:46843} 2024-11-09T05:40:17,279 INFO [Time-limited test {}] server.Server(415): Started @4462ms 2024-11-09T05:40:17,283 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-09T05:40:17,368 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T05:40:17,378 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T05:40:17,391 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T05:40:17,392 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T05:40:17,392 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-09T05:40:17,393 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/hadoop.log.dir/,AVAILABLE} 2024-11-09T05:40:17,394 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T05:40:17,489 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/cluster_a63c0870-ce87-d57c-f3a1-ecd7f1f7e9af/data/data2/current/BP-1990629608-172.17.0.2-1731130815413/current, will proceed with Du for space computation calculation, 2024-11-09T05:40:17,489 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/cluster_a63c0870-ce87-d57c-f3a1-ecd7f1f7e9af/data/data3/current/BP-1990629608-172.17.0.2-1731130815413/current, will proceed with Du for space computation calculation, 2024-11-09T05:40:17,490 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/cluster_a63c0870-ce87-d57c-f3a1-ecd7f1f7e9af/data/data1/current/BP-1990629608-172.17.0.2-1731130815413/current, will proceed with Du for space computation calculation, 2024-11-09T05:40:17,490 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/cluster_a63c0870-ce87-d57c-f3a1-ecd7f1f7e9af/data/data4/current/BP-1990629608-172.17.0.2-1731130815413/current, will proceed with Du for space computation calculation, 2024-11-09T05:40:17,573 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-09T05:40:17,576 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-09T05:40:17,568 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/java.io.tmpdir/jetty-localhost-35641-hadoop-hdfs-3_4_1-tests_jar-_-any-14458203662549307060/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T05:40:17,582 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:35641} 2024-11-09T05:40:17,582 INFO [Time-limited test {}] server.Server(415): Started @4764ms 2024-11-09T05:40:17,587 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-09T05:40:17,685 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x874bf2fb8c12f8d9 with lease ID 0x914fcf15e21f3afe: Processing first storage report for DS-ed00c975-41ab-41f9-90d6-754aff306015 from datanode DatanodeRegistration(127.0.0.1:43869, datanodeUuid=9491a6d9-0005-421a-903d-c50907f25a07, infoPort=44645, infoSecurePort=0, ipcPort=43739, storageInfo=lv=-57;cid=testClusterID;nsid=767191375;c=1731130815413) 2024-11-09T05:40:17,687 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x874bf2fb8c12f8d9 with lease ID 0x914fcf15e21f3afe: from storage DS-ed00c975-41ab-41f9-90d6-754aff306015 node DatanodeRegistration(127.0.0.1:43869, datanodeUuid=9491a6d9-0005-421a-903d-c50907f25a07, infoPort=44645, infoSecurePort=0, ipcPort=43739, storageInfo=lv=-57;cid=testClusterID;nsid=767191375;c=1731130815413), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-09T05:40:17,687 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6b1656116f5b8e8f with lease ID 0x914fcf15e21f3aff: Processing first storage report for DS-0dfa7937-2ca7-4bc6-8d88-b6076a7fadac from datanode DatanodeRegistration(127.0.0.1:39285, datanodeUuid=2bb7a50f-c3c3-43f6-9175-ef752bd980de, infoPort=42387, infoSecurePort=0, ipcPort=39273, storageInfo=lv=-57;cid=testClusterID;nsid=767191375;c=1731130815413) 2024-11-09T05:40:17,687 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6b1656116f5b8e8f with lease ID 0x914fcf15e21f3aff: from storage DS-0dfa7937-2ca7-4bc6-8d88-b6076a7fadac node DatanodeRegistration(127.0.0.1:39285, datanodeUuid=2bb7a50f-c3c3-43f6-9175-ef752bd980de, infoPort=42387, infoSecurePort=0, ipcPort=39273, storageInfo=lv=-57;cid=testClusterID;nsid=767191375;c=1731130815413), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-09T05:40:17,688 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x874bf2fb8c12f8d9 with lease ID 0x914fcf15e21f3afe: Processing first storage report for DS-e09793f9-0264-4217-9042-5c3531694a8b from datanode DatanodeRegistration(127.0.0.1:43869, datanodeUuid=9491a6d9-0005-421a-903d-c50907f25a07, infoPort=44645, infoSecurePort=0, ipcPort=43739, storageInfo=lv=-57;cid=testClusterID;nsid=767191375;c=1731130815413) 2024-11-09T05:40:17,688 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x874bf2fb8c12f8d9 with lease ID 0x914fcf15e21f3afe: from storage DS-e09793f9-0264-4217-9042-5c3531694a8b node DatanodeRegistration(127.0.0.1:43869, datanodeUuid=9491a6d9-0005-421a-903d-c50907f25a07, infoPort=44645, infoSecurePort=0, ipcPort=43739, storageInfo=lv=-57;cid=testClusterID;nsid=767191375;c=1731130815413), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-09T05:40:17,688 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6b1656116f5b8e8f with lease ID 0x914fcf15e21f3aff: Processing first storage report for DS-a261e5f8-800e-4490-9cd2-6e76f8c3a690 from datanode DatanodeRegistration(127.0.0.1:39285, datanodeUuid=2bb7a50f-c3c3-43f6-9175-ef752bd980de, infoPort=42387, infoSecurePort=0, ipcPort=39273, storageInfo=lv=-57;cid=testClusterID;nsid=767191375;c=1731130815413) 2024-11-09T05:40:17,689 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6b1656116f5b8e8f with lease ID 0x914fcf15e21f3aff: from storage DS-a261e5f8-800e-4490-9cd2-6e76f8c3a690 node DatanodeRegistration(127.0.0.1:39285, datanodeUuid=2bb7a50f-c3c3-43f6-9175-ef752bd980de, infoPort=42387, infoSecurePort=0, ipcPort=39273, storageInfo=lv=-57;cid=testClusterID;nsid=767191375;c=1731130815413), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-09T05:40:17,836 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/cluster_a63c0870-ce87-d57c-f3a1-ecd7f1f7e9af/data/data6/current/BP-1990629608-172.17.0.2-1731130815413/current, will proceed with Du for space computation calculation, 2024-11-09T05:40:17,840 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/cluster_a63c0870-ce87-d57c-f3a1-ecd7f1f7e9af/data/data5/current/BP-1990629608-172.17.0.2-1731130815413/current, will proceed with Du for space computation calculation, 2024-11-09T05:40:17,910 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-09T05:40:17,921 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x852c176ff82862a5 with lease ID 0x914fcf15e21f3b00: Processing first storage report for DS-8282eccd-3379-4ce2-9612-3e9a0257084e from datanode DatanodeRegistration(127.0.0.1:34201, datanodeUuid=c9f51109-a2c6-4f25-ba0d-47a4528b65ae, infoPort=46203, infoSecurePort=0, ipcPort=40893, storageInfo=lv=-57;cid=testClusterID;nsid=767191375;c=1731130815413) 2024-11-09T05:40:17,921 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x852c176ff82862a5 with lease ID 0x914fcf15e21f3b00: from storage DS-8282eccd-3379-4ce2-9612-3e9a0257084e node DatanodeRegistration(127.0.0.1:34201, datanodeUuid=c9f51109-a2c6-4f25-ba0d-47a4528b65ae, infoPort=46203, infoSecurePort=0, ipcPort=40893, storageInfo=lv=-57;cid=testClusterID;nsid=767191375;c=1731130815413), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-09T05:40:17,922 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x852c176ff82862a5 with lease ID 0x914fcf15e21f3b00: Processing first storage report for DS-d242637c-9bd1-4714-8742-beaab78b87bd from datanode DatanodeRegistration(127.0.0.1:34201, datanodeUuid=c9f51109-a2c6-4f25-ba0d-47a4528b65ae, infoPort=46203, infoSecurePort=0, ipcPort=40893, storageInfo=lv=-57;cid=testClusterID;nsid=767191375;c=1731130815413) 2024-11-09T05:40:17,922 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x852c176ff82862a5 with lease ID 0x914fcf15e21f3b00: from storage DS-d242637c-9bd1-4714-8742-beaab78b87bd node DatanodeRegistration(127.0.0.1:34201, datanodeUuid=c9f51109-a2c6-4f25-ba0d-47a4528b65ae, infoPort=46203, infoSecurePort=0, ipcPort=40893, storageInfo=lv=-57;cid=testClusterID;nsid=767191375;c=1731130815413), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-09T05:40:18,158 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82 2024-11-09T05:40:18,255 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/cluster_a63c0870-ce87-d57c-f3a1-ecd7f1f7e9af/zookeeper_0, clientPort=51298, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/cluster_a63c0870-ce87-d57c-f3a1-ecd7f1f7e9af/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c9d0ec94-fe45-b413-5351-9346409e6b82/cluster_a63c0870-ce87-d57c-f3a1-ecd7f1f7e9af/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-09T05:40:18,268 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51298 2024-11-09T05:40:18,283 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T05:40:18,286 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T05:40:18,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741825_1001 (size=7) 2024-11-09T05:40:18,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741825_1001 (size=7) 2024-11-09T05:40:18,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741825_1001 (size=7) 2024-11-09T05:40:18,951 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 with version=8 2024-11-09T05:40:18,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/hbase-staging 2024-11-09T05:40:19,050 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-09T05:40:19,329 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e09398052c91:0 server-side Connection retries=45 2024-11-09T05:40:19,344 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T05:40:19,345 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T05:40:19,352 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T05:40:19,353 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T05:40:19,353 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T05:40:19,522 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-09T05:40:19,596 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-09T05:40:19,609 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-09T05:40:19,615 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T05:40:19,648 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1009 (auto-detected) 2024-11-09T05:40:19,649 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-09T05:40:19,675 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34809 2024-11-09T05:40:19,700 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34809 connecting to ZooKeeper ensemble=127.0.0.1:51298 2024-11-09T05:40:19,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:348090x0, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T05:40:19,743 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34809-0x100fb3de9450000 connected 2024-11-09T05:40:19,793 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T05:40:19,798 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T05:40:19,809 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T05:40:19,814 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352, hbase.cluster.distributed=false 2024-11-09T05:40:19,846 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T05:40:19,861 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34809 2024-11-09T05:40:19,862 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34809 2024-11-09T05:40:19,863 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34809 2024-11-09T05:40:19,864 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34809 2024-11-09T05:40:19,864 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34809 2024-11-09T05:40:20,005 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e09398052c91:0 server-side Connection retries=45 2024-11-09T05:40:20,007 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T05:40:20,007 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T05:40:20,007 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T05:40:20,007 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T05:40:20,008 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T05:40:20,011 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T05:40:20,014 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T05:40:20,015 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44035 2024-11-09T05:40:20,017 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44035 connecting to ZooKeeper ensemble=127.0.0.1:51298 2024-11-09T05:40:20,018 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T05:40:20,021 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T05:40:20,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:440350x0, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T05:40:20,029 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44035-0x100fb3de9450001 connected 2024-11-09T05:40:20,029 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T05:40:20,034 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T05:40:20,043 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T05:40:20,047 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T05:40:20,053 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T05:40:20,054 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44035 2024-11-09T05:40:20,055 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44035 2024-11-09T05:40:20,055 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44035 2024-11-09T05:40:20,056 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44035 2024-11-09T05:40:20,057 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44035 2024-11-09T05:40:20,077 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e09398052c91:0 server-side Connection retries=45 2024-11-09T05:40:20,077 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T05:40:20,077 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T05:40:20,078 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T05:40:20,078 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T05:40:20,078 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T05:40:20,078 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T05:40:20,078 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T05:40:20,079 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34331 2024-11-09T05:40:20,081 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34331 connecting to ZooKeeper ensemble=127.0.0.1:51298 2024-11-09T05:40:20,082 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T05:40:20,085 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T05:40:20,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:343310x0, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T05:40:20,093 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34331-0x100fb3de9450002 connected 2024-11-09T05:40:20,095 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34331-0x100fb3de9450002, quorum=127.0.0.1:51298, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T05:40:20,095 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T05:40:20,098 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T05:40:20,100 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34331-0x100fb3de9450002, quorum=127.0.0.1:51298, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T05:40:20,102 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34331-0x100fb3de9450002, quorum=127.0.0.1:51298, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T05:40:20,102 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34331 2024-11-09T05:40:20,103 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34331 2024-11-09T05:40:20,104 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34331 2024-11-09T05:40:20,109 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34331 2024-11-09T05:40:20,109 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34331 2024-11-09T05:40:20,129 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e09398052c91:0 server-side Connection retries=45 2024-11-09T05:40:20,129 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T05:40:20,129 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T05:40:20,130 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T05:40:20,130 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T05:40:20,130 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T05:40:20,130 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T05:40:20,130 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T05:40:20,133 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40411 2024-11-09T05:40:20,135 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40411 connecting to ZooKeeper ensemble=127.0.0.1:51298 2024-11-09T05:40:20,136 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T05:40:20,139 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T05:40:20,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:404110x0, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T05:40:20,146 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:404110x0, quorum=127.0.0.1:51298, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T05:40:20,146 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40411-0x100fb3de9450003 connected 2024-11-09T05:40:20,147 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T05:40:20,150 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T05:40:20,152 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T05:40:20,156 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T05:40:20,157 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40411 2024-11-09T05:40:20,160 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40411 2024-11-09T05:40:20,167 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40411 2024-11-09T05:40:20,168 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40411 2024-11-09T05:40:20,168 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40411 2024-11-09T05:40:20,183 DEBUG [M:0;e09398052c91:34809 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e09398052c91:34809 2024-11-09T05:40:20,185 INFO [master/e09398052c91:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e09398052c91,34809,1731130819107 2024-11-09T05:40:20,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T05:40:20,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T05:40:20,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T05:40:20,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34331-0x100fb3de9450002, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T05:40:20,196 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e09398052c91,34809,1731130819107 2024-11-09T05:40:20,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T05:40:20,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T05:40:20,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34331-0x100fb3de9450002, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T05:40:20,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:20,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:20,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34331-0x100fb3de9450002, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:20,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:20,226 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-09T05:40:20,229 INFO [master/e09398052c91:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e09398052c91,34809,1731130819107 from backup master directory 2024-11-09T05:40:20,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34331-0x100fb3de9450002, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T05:40:20,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T05:40:20,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T05:40:20,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e09398052c91,34809,1731130819107 2024-11-09T05:40:20,233 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T05:40:20,234 WARN [master/e09398052c91:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T05:40:20,234 INFO [master/e09398052c91:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e09398052c91,34809,1731130819107 2024-11-09T05:40:20,238 INFO [master/e09398052c91:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-09T05:40:20,239 INFO [master/e09398052c91:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-09T05:40:20,302 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/hbase.id] with ID: eb26d979-02fa-4bab-82e5-c5ab4630e8d4 2024-11-09T05:40:20,302 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/.tmp/hbase.id 2024-11-09T05:40:20,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741826_1002 (size=42) 2024-11-09T05:40:20,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741826_1002 (size=42) 2024-11-09T05:40:20,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741826_1002 (size=42) 2024-11-09T05:40:20,323 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/.tmp/hbase.id]:[hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/hbase.id] 2024-11-09T05:40:20,373 INFO [master/e09398052c91:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T05:40:20,379 INFO [master/e09398052c91:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-09T05:40:20,401 INFO [master/e09398052c91:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-09T05:40:20,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:20,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:20,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:20,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34331-0x100fb3de9450002, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:20,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741827_1003 (size=196) 2024-11-09T05:40:20,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741827_1003 (size=196) 2024-11-09T05:40:20,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741827_1003 (size=196) 2024-11-09T05:40:20,457 INFO [master/e09398052c91:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T05:40:20,459 INFO [master/e09398052c91:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-09T05:40:20,480 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T05:40:20,486 INFO [master/e09398052c91:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T05:40:20,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741828_1004 (size=1189) 2024-11-09T05:40:20,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741828_1004 (size=1189) 2024-11-09T05:40:20,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741828_1004 (size=1189) 2024-11-09T05:40:20,550 INFO [master/e09398052c91:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/MasterData/data/master/store 2024-11-09T05:40:20,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741829_1005 (size=34) 2024-11-09T05:40:20,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741829_1005 (size=34) 2024-11-09T05:40:20,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741829_1005 (size=34) 2024-11-09T05:40:20,587 INFO [master/e09398052c91:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-09T05:40:20,591 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:20,593 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T05:40:20,593 INFO [master/e09398052c91:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T05:40:20,593 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T05:40:20,596 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T05:40:20,596 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T05:40:20,596 INFO [master/e09398052c91:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T05:40:20,598 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731130820592Disabling compacts and flushes for region at 1731130820592Disabling writes for close at 1731130820596 (+4 ms)Writing region close event to WAL at 1731130820596Closed at 1731130820596 2024-11-09T05:40:20,600 WARN [master/e09398052c91:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/MasterData/data/master/store/.initializing 2024-11-09T05:40:20,601 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/MasterData/WALs/e09398052c91,34809,1731130819107 2024-11-09T05:40:20,610 INFO [master/e09398052c91:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T05:40:20,630 INFO [master/e09398052c91:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e09398052c91%2C34809%2C1731130819107, suffix=, logDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/MasterData/WALs/e09398052c91,34809,1731130819107, archiveDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/MasterData/oldWALs, maxLogs=10 2024-11-09T05:40:20,657 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/MasterData/WALs/e09398052c91,34809,1731130819107/e09398052c91%2C34809%2C1731130819107.1731130820635, exclude list is [], retry=0 2024-11-09T05:40:20,661 WARN [IPC Server handler 2 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:20,662 WARN [IPC Server handler 2 on default port 43215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T05:40:20,662 WARN [IPC Server handler 2 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T05:40:20,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43869,DS-ed00c975-41ab-41f9-90d6-754aff306015,DISK] 2024-11-09T05:40:20,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34201,DS-8282eccd-3379-4ce2-9612-3e9a0257084e,DISK] 2024-11-09T05:40:20,683 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-09T05:40:20,731 INFO [master/e09398052c91:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/MasterData/WALs/e09398052c91,34809,1731130819107/e09398052c91%2C34809%2C1731130819107.1731130820635 2024-11-09T05:40:20,733 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46203:46203),(127.0.0.1/127.0.0.1:44645:44645)] 2024-11-09T05:40:20,733 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-09T05:40:20,734 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:20,737 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T05:40:20,739 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T05:40:20,810 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T05:40:20,841 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-09T05:40:20,845 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:20,849 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T05:40:20,850 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T05:40:20,854 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-09T05:40:20,854 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:20,856 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:20,856 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T05:40:20,861 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-09T05:40:20,861 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:20,862 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:20,863 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T05:40:20,865 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-09T05:40:20,866 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:20,867 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:20,867 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T05:40:20,872 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T05:40:20,873 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T05:40:20,882 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T05:40:20,882 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T05:40:20,886 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T05:40:20,891 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T05:40:20,897 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:20,899 INFO [master/e09398052c91:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67494228, jitterRate=0.005742371082305908}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T05:40:20,911 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731130820764Initializing all the Stores at 1731130820768 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731130820769 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130820771 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130820771Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130820772 (+1 ms)Cleaning up temporary data from old regions at 1731130820882 (+110 ms)Region opened successfully at 1731130820911 (+29 ms) 2024-11-09T05:40:20,913 INFO [master/e09398052c91:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-09T05:40:20,953 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f62c3b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e09398052c91/172.17.0.2:0 2024-11-09T05:40:20,988 INFO [master/e09398052c91:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-09T05:40:21,000 INFO [master/e09398052c91:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:21,001 INFO [master/e09398052c91:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:21,004 INFO [master/e09398052c91:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:21,006 INFO [master/e09398052c91:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-09T05:40:21,011 INFO [master/e09398052c91:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-09T05:40:21,011 INFO [master/e09398052c91:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-09T05:40:21,041 INFO [master/e09398052c91:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-09T05:40:21,049 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-09T05:40:21,051 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-09T05:40:21,053 INFO [master/e09398052c91:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-09T05:40:21,055 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-09T05:40:21,056 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-09T05:40:21,058 INFO [master/e09398052c91:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-09T05:40:21,061 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-09T05:40:21,063 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-09T05:40:21,064 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-09T05:40:21,067 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-09T05:40:21,085 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-09T05:40:21,087 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-09T05:40:21,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T05:40:21,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34331-0x100fb3de9450002, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T05:40:21,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:21,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34331-0x100fb3de9450002, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:21,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T05:40:21,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:21,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T05:40:21,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:21,094 INFO [master/e09398052c91:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e09398052c91,34809,1731130819107, sessionid=0x100fb3de9450000, setting cluster-up flag (Was=false) 2024-11-09T05:40:21,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:21,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:21,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34331-0x100fb3de9450002, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:21,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:21,116 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-09T05:40:21,118 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e09398052c91,34809,1731130819107 2024-11-09T05:40:21,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:21,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:21,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34331-0x100fb3de9450002, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:21,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:21,137 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-09T05:40:21,139 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e09398052c91,34809,1731130819107 2024-11-09T05:40:21,146 INFO [master/e09398052c91:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-09T05:40:21,173 INFO [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(746): ClusterId : eb26d979-02fa-4bab-82e5-c5ab4630e8d4 2024-11-09T05:40:21,173 INFO [RS:1;e09398052c91:34331 {}] regionserver.HRegionServer(746): ClusterId : eb26d979-02fa-4bab-82e5-c5ab4630e8d4 2024-11-09T05:40:21,176 INFO [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(746): ClusterId : eb26d979-02fa-4bab-82e5-c5ab4630e8d4 2024-11-09T05:40:21,177 DEBUG [RS:2;e09398052c91:40411 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T05:40:21,177 DEBUG [RS:1;e09398052c91:34331 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T05:40:21,177 DEBUG [RS:0;e09398052c91:44035 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T05:40:21,186 DEBUG [RS:2;e09398052c91:40411 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T05:40:21,186 DEBUG [RS:2;e09398052c91:40411 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T05:40:21,187 DEBUG [RS:0;e09398052c91:44035 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T05:40:21,187 DEBUG [RS:0;e09398052c91:44035 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T05:40:21,188 DEBUG [RS:1;e09398052c91:34331 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T05:40:21,188 DEBUG [RS:1;e09398052c91:34331 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T05:40:21,191 DEBUG [RS:2;e09398052c91:40411 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T05:40:21,192 DEBUG [RS:0;e09398052c91:44035 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T05:40:21,192 DEBUG [RS:2;e09398052c91:40411 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@530da76a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e09398052c91/172.17.0.2:0 2024-11-09T05:40:21,192 DEBUG [RS:1;e09398052c91:34331 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T05:40:21,192 DEBUG [RS:0;e09398052c91:44035 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30eda7d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e09398052c91/172.17.0.2:0 2024-11-09T05:40:21,193 DEBUG [RS:1;e09398052c91:34331 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@890ee7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e09398052c91/172.17.0.2:0 2024-11-09T05:40:21,220 DEBUG [RS:2;e09398052c91:40411 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;e09398052c91:40411 2024-11-09T05:40:21,220 DEBUG [RS:1;e09398052c91:34331 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;e09398052c91:34331 2024-11-09T05:40:21,226 DEBUG [RS:0;e09398052c91:44035 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e09398052c91:44035 2024-11-09T05:40:21,227 INFO [RS:2;e09398052c91:40411 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T05:40:21,228 INFO [RS:2;e09398052c91:40411 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T05:40:21,228 DEBUG [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T05:40:21,228 INFO [RS:1;e09398052c91:34331 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T05:40:21,228 INFO [RS:1;e09398052c91:34331 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T05:40:21,228 DEBUG [RS:1;e09398052c91:34331 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T05:40:21,229 INFO [RS:0;e09398052c91:44035 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T05:40:21,229 INFO [RS:0;e09398052c91:44035 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T05:40:21,229 DEBUG [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T05:40:21,233 INFO [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(2659): reportForDuty to master=e09398052c91,34809,1731130819107 with port=40411, startcode=1731130820128 2024-11-09T05:40:21,234 INFO [RS:1;e09398052c91:34331 {}] regionserver.HRegionServer(2659): reportForDuty to master=e09398052c91,34809,1731130819107 with port=34331, startcode=1731130820076 2024-11-09T05:40:21,234 INFO [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(2659): reportForDuty to master=e09398052c91,34809,1731130819107 with port=44035, startcode=1731130819959 2024-11-09T05:40:21,250 DEBUG [RS:2;e09398052c91:40411 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T05:40:21,250 DEBUG [RS:0;e09398052c91:44035 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T05:40:21,250 DEBUG [RS:1;e09398052c91:34331 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T05:40:21,254 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:21,268 INFO [master/e09398052c91:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-09T05:40:21,279 INFO [master/e09398052c91:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-09T05:40:21,286 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e09398052c91,34809,1731130819107 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-09T05:40:21,295 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e09398052c91:0, corePoolSize=5, maxPoolSize=5 2024-11-09T05:40:21,296 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e09398052c91:0, corePoolSize=5, maxPoolSize=5 2024-11-09T05:40:21,296 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e09398052c91:0, corePoolSize=5, maxPoolSize=5 2024-11-09T05:40:21,296 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e09398052c91:0, corePoolSize=5, maxPoolSize=5 2024-11-09T05:40:21,296 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e09398052c91:0, corePoolSize=10, maxPoolSize=10 2024-11-09T05:40:21,296 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,296 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e09398052c91:0, corePoolSize=2, maxPoolSize=2 2024-11-09T05:40:21,297 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,301 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54859, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T05:40:21,301 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39631, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T05:40:21,301 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37261, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T05:40:21,306 INFO [master/e09398052c91:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130851305 2024-11-09T05:40:21,307 INFO [master/e09398052c91:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-09T05:40:21,308 INFO [master/e09398052c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-09T05:40:21,308 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:21,311 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:21,312 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-09T05:40:21,313 INFO [master/e09398052c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-09T05:40:21,313 INFO [master/e09398052c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-09T05:40:21,314 INFO [master/e09398052c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-09T05:40:21,314 INFO [master/e09398052c91:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-09T05:40:21,316 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:21,317 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:21,316 INFO [master/e09398052c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,322 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:21,322 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-09T05:40:21,325 INFO [master/e09398052c91:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-09T05:40:21,326 INFO [master/e09398052c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-09T05:40:21,326 INFO [master/e09398052c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-09T05:40:21,329 INFO [master/e09398052c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-09T05:40:21,329 INFO [master/e09398052c91:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-09T05:40:21,330 WARN [IPC Server handler 2 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:21,330 WARN [IPC Server handler 2 on default port 43215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T05:40:21,330 WARN [IPC Server handler 2 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T05:40:21,332 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e09398052c91:0:becomeActiveMaster-HFileCleaner.large.0-1731130821330,5,FailOnTimeoutGroup] 2024-11-09T05:40:21,336 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e09398052c91:0:becomeActiveMaster-HFileCleaner.small.0-1731130821332,5,FailOnTimeoutGroup] 2024-11-09T05:40:21,337 INFO [master/e09398052c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,337 INFO [master/e09398052c91:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-09T05:40:21,339 INFO [master/e09398052c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,340 INFO [master/e09398052c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,341 DEBUG [RS:1;e09398052c91:34331 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-09T05:40:21,341 DEBUG [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-09T05:40:21,341 DEBUG [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-09T05:40:21,342 WARN [RS:1;e09398052c91:34331 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-09T05:40:21,342 WARN [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-09T05:40:21,342 WARN [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-09T05:40:21,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741831_1007 (size=1321) 2024-11-09T05:40:21,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741831_1007 (size=1321) 2024-11-09T05:40:21,353 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-09T05:40:21,354 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:21,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741832_1008 (size=32) 2024-11-09T05:40:21,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741832_1008 (size=32) 2024-11-09T05:40:21,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741832_1008 (size=32) 2024-11-09T05:40:21,378 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:21,380 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T05:40:21,382 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T05:40:21,383 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:21,383 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T05:40:21,384 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T05:40:21,386 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T05:40:21,386 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:21,387 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T05:40:21,387 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T05:40:21,389 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T05:40:21,389 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:21,390 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T05:40:21,390 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T05:40:21,392 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T05:40:21,393 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:21,393 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T05:40:21,394 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T05:40:21,395 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740 2024-11-09T05:40:21,396 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740 2024-11-09T05:40:21,399 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T05:40:21,399 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T05:40:21,400 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T05:40:21,403 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T05:40:21,407 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:21,408 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74316644, jitterRate=0.10740429162979126}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T05:40:21,411 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731130821378Initializing all the Stores at 1731130821379 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731130821380 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731130821380Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130821380Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731130821380Cleaning up temporary data from old regions at 1731130821399 (+19 ms)Region opened successfully at 1731130821411 (+12 ms) 2024-11-09T05:40:21,411 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T05:40:21,411 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T05:40:21,411 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T05:40:21,411 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T05:40:21,411 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T05:40:21,413 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T05:40:21,413 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731130821411Disabling compacts and flushes for region at 1731130821411Disabling writes for close at 1731130821411Writing region close event to WAL at 1731130821412 (+1 ms)Closed at 1731130821413 (+1 ms) 2024-11-09T05:40:21,417 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:21,417 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-09T05:40:21,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-09T05:40:21,434 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:21,437 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-09T05:40:21,443 INFO [RS:1;e09398052c91:34331 {}] regionserver.HRegionServer(2659): reportForDuty to master=e09398052c91,34809,1731130819107 with port=34331, startcode=1731130820076 2024-11-09T05:40:21,443 INFO [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(2659): reportForDuty to master=e09398052c91,34809,1731130819107 with port=44035, startcode=1731130819959 2024-11-09T05:40:21,444 INFO [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(2659): reportForDuty to master=e09398052c91,34809,1731130819107 with port=40411, startcode=1731130820128 2024-11-09T05:40:21,445 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e09398052c91,44035,1731130819959 2024-11-09T05:40:21,448 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] master.ServerManager(517): Registering regionserver=e09398052c91,44035,1731130819959 2024-11-09T05:40:21,457 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e09398052c91,34331,1731130820076 2024-11-09T05:40:21,457 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] master.ServerManager(517): Registering regionserver=e09398052c91,34331,1731130820076 2024-11-09T05:40:21,457 DEBUG [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:21,457 DEBUG [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43215 2024-11-09T05:40:21,457 DEBUG [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T05:40:21,460 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e09398052c91,40411,1731130820128 2024-11-09T05:40:21,460 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] master.ServerManager(517): Registering regionserver=e09398052c91,40411,1731130820128 2024-11-09T05:40:21,460 DEBUG [RS:1;e09398052c91:34331 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:21,460 DEBUG [RS:1;e09398052c91:34331 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43215 2024-11-09T05:40:21,460 DEBUG [RS:1;e09398052c91:34331 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T05:40:21,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T05:40:21,464 DEBUG [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:21,464 DEBUG [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43215 2024-11-09T05:40:21,464 DEBUG [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T05:40:21,467 DEBUG [RS:0;e09398052c91:44035 {}] zookeeper.ZKUtil(111): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e09398052c91,44035,1731130819959 2024-11-09T05:40:21,467 WARN [RS:0;e09398052c91:44035 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T05:40:21,467 INFO [RS:0;e09398052c91:44035 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T05:40:21,467 DEBUG [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,44035,1731130819959 2024-11-09T05:40:21,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T05:40:21,469 DEBUG [RS:1;e09398052c91:34331 {}] zookeeper.ZKUtil(111): regionserver:34331-0x100fb3de9450002, quorum=127.0.0.1:51298, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e09398052c91,34331,1731130820076 2024-11-09T05:40:21,469 WARN [RS:1;e09398052c91:34331 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T05:40:21,469 INFO [RS:1;e09398052c91:34331 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T05:40:21,469 DEBUG [RS:1;e09398052c91:34331 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076 2024-11-09T05:40:21,469 DEBUG [RS:2;e09398052c91:40411 {}] zookeeper.ZKUtil(111): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e09398052c91,40411,1731130820128 2024-11-09T05:40:21,472 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e09398052c91,34331,1731130820076] 2024-11-09T05:40:21,472 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e09398052c91,44035,1731130819959] 2024-11-09T05:40:21,472 WARN [RS:2;e09398052c91:40411 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T05:40:21,473 INFO [RS:2;e09398052c91:40411 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T05:40:21,473 DEBUG [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,40411,1731130820128 2024-11-09T05:40:21,473 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e09398052c91,40411,1731130820128] 2024-11-09T05:40:21,499 INFO [RS:0;e09398052c91:44035 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T05:40:21,499 INFO [RS:1;e09398052c91:34331 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T05:40:21,499 INFO [RS:2;e09398052c91:40411 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T05:40:21,513 INFO [RS:1;e09398052c91:34331 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T05:40:21,513 INFO [RS:2;e09398052c91:40411 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T05:40:21,513 INFO [RS:0;e09398052c91:44035 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T05:40:21,520 INFO [RS:2;e09398052c91:40411 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T05:40:21,520 INFO [RS:0;e09398052c91:44035 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T05:40:21,520 INFO [RS:1;e09398052c91:34331 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T05:40:21,520 INFO [RS:2;e09398052c91:40411 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,520 INFO [RS:1;e09398052c91:34331 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,520 INFO [RS:0;e09398052c91:44035 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,521 INFO [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T05:40:21,521 INFO [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T05:40:21,523 INFO [RS:1;e09398052c91:34331 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T05:40:21,527 INFO [RS:0;e09398052c91:44035 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T05:40:21,527 INFO [RS:1;e09398052c91:34331 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T05:40:21,527 INFO [RS:2;e09398052c91:40411 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T05:40:21,528 INFO [RS:0;e09398052c91:44035 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,528 INFO [RS:1;e09398052c91:34331 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,528 INFO [RS:2;e09398052c91:40411 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,529 DEBUG [RS:2;e09398052c91:40411 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,529 DEBUG [RS:1;e09398052c91:34331 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,529 DEBUG [RS:0;e09398052c91:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,529 DEBUG [RS:2;e09398052c91:40411 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,529 DEBUG [RS:0;e09398052c91:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,529 DEBUG [RS:1;e09398052c91:34331 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,529 DEBUG [RS:2;e09398052c91:40411 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,529 DEBUG [RS:0;e09398052c91:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,529 DEBUG [RS:1;e09398052c91:34331 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,529 DEBUG [RS:1;e09398052c91:34331 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,529 DEBUG [RS:0;e09398052c91:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,529 DEBUG [RS:2;e09398052c91:40411 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,529 DEBUG [RS:1;e09398052c91:34331 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,529 DEBUG [RS:0;e09398052c91:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,529 DEBUG [RS:2;e09398052c91:40411 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,530 DEBUG [RS:0;e09398052c91:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0, corePoolSize=2, maxPoolSize=2 2024-11-09T05:40:21,530 DEBUG [RS:1;e09398052c91:34331 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0, corePoolSize=2, maxPoolSize=2 2024-11-09T05:40:21,530 DEBUG [RS:2;e09398052c91:40411 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0, corePoolSize=2, maxPoolSize=2 2024-11-09T05:40:21,530 DEBUG [RS:1;e09398052c91:34331 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,530 DEBUG [RS:0;e09398052c91:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,530 DEBUG [RS:2;e09398052c91:40411 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,530 DEBUG [RS:0;e09398052c91:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,530 DEBUG [RS:1;e09398052c91:34331 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,530 DEBUG [RS:2;e09398052c91:40411 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,530 DEBUG [RS:0;e09398052c91:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,530 DEBUG [RS:1;e09398052c91:34331 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,530 DEBUG [RS:2;e09398052c91:40411 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,530 DEBUG [RS:0;e09398052c91:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,530 DEBUG [RS:1;e09398052c91:34331 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,530 DEBUG [RS:2;e09398052c91:40411 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,530 DEBUG [RS:0;e09398052c91:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,530 DEBUG [RS:1;e09398052c91:34331 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,530 DEBUG [RS:2;e09398052c91:40411 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,531 DEBUG [RS:0;e09398052c91:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,531 DEBUG [RS:1;e09398052c91:34331 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,531 DEBUG [RS:2;e09398052c91:40411 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e09398052c91:0, corePoolSize=1, maxPoolSize=1 2024-11-09T05:40:21,531 DEBUG [RS:1;e09398052c91:34331 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e09398052c91:0, corePoolSize=3, maxPoolSize=3 2024-11-09T05:40:21,531 DEBUG [RS:0;e09398052c91:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e09398052c91:0, corePoolSize=3, maxPoolSize=3 2024-11-09T05:40:21,531 DEBUG [RS:2;e09398052c91:40411 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e09398052c91:0, corePoolSize=3, maxPoolSize=3 2024-11-09T05:40:21,531 DEBUG [RS:0;e09398052c91:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e09398052c91:0, corePoolSize=3, maxPoolSize=3 2024-11-09T05:40:21,531 DEBUG [RS:1;e09398052c91:34331 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e09398052c91:0, corePoolSize=3, maxPoolSize=3 2024-11-09T05:40:21,531 DEBUG [RS:2;e09398052c91:40411 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e09398052c91:0, corePoolSize=3, maxPoolSize=3 2024-11-09T05:40:21,532 INFO [RS:2;e09398052c91:40411 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,532 INFO [RS:2;e09398052c91:40411 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,532 INFO [RS:2;e09398052c91:40411 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,533 INFO [RS:0;e09398052c91:44035 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,533 INFO [RS:2;e09398052c91:40411 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,533 INFO [RS:0;e09398052c91:44035 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,533 INFO [RS:1;e09398052c91:34331 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,533 INFO [RS:2;e09398052c91:40411 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,533 INFO [RS:0;e09398052c91:44035 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,533 INFO [RS:1;e09398052c91:34331 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,533 INFO [RS:2;e09398052c91:40411 {}] hbase.ChoreService(168): Chore ScheduledChore name=e09398052c91,40411,1731130820128-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T05:40:21,533 INFO [RS:0;e09398052c91:44035 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,533 INFO [RS:1;e09398052c91:34331 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,533 INFO [RS:0;e09398052c91:44035 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,533 INFO [RS:1;e09398052c91:34331 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,533 INFO [RS:0;e09398052c91:44035 {}] hbase.ChoreService(168): Chore ScheduledChore name=e09398052c91,44035,1731130819959-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T05:40:21,533 INFO [RS:1;e09398052c91:34331 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,533 INFO [RS:1;e09398052c91:34331 {}] hbase.ChoreService(168): Chore ScheduledChore name=e09398052c91,34331,1731130820076-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T05:40:21,554 INFO [RS:0;e09398052c91:44035 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T05:40:21,554 INFO [RS:2;e09398052c91:40411 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T05:40:21,556 INFO [RS:0;e09398052c91:44035 {}] hbase.ChoreService(168): Chore ScheduledChore name=e09398052c91,44035,1731130819959-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,556 INFO [RS:2;e09398052c91:40411 {}] hbase.ChoreService(168): Chore ScheduledChore name=e09398052c91,40411,1731130820128-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,557 INFO [RS:0;e09398052c91:44035 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,557 INFO [RS:2;e09398052c91:40411 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,557 INFO [RS:0;e09398052c91:44035 {}] regionserver.Replication(171): e09398052c91,44035,1731130819959 started 2024-11-09T05:40:21,557 INFO [RS:2;e09398052c91:40411 {}] regionserver.Replication(171): e09398052c91,40411,1731130820128 started 2024-11-09T05:40:21,559 INFO [RS:1;e09398052c91:34331 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T05:40:21,559 INFO [RS:1;e09398052c91:34331 {}] hbase.ChoreService(168): Chore ScheduledChore name=e09398052c91,34331,1731130820076-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,560 INFO [RS:1;e09398052c91:34331 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,560 INFO [RS:1;e09398052c91:34331 {}] regionserver.Replication(171): e09398052c91,34331,1731130820076 started 2024-11-09T05:40:21,574 INFO [RS:0;e09398052c91:44035 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,575 INFO [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(1482): Serving as e09398052c91,44035,1731130819959, RpcServer on e09398052c91/172.17.0.2:44035, sessionid=0x100fb3de9450001 2024-11-09T05:40:21,575 INFO [RS:2;e09398052c91:40411 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,575 INFO [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(1482): Serving as e09398052c91,40411,1731130820128, RpcServer on e09398052c91/172.17.0.2:40411, sessionid=0x100fb3de9450003 2024-11-09T05:40:21,576 DEBUG [RS:0;e09398052c91:44035 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T05:40:21,576 DEBUG [RS:2;e09398052c91:40411 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T05:40:21,576 DEBUG [RS:0;e09398052c91:44035 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e09398052c91,44035,1731130819959 2024-11-09T05:40:21,576 DEBUG [RS:2;e09398052c91:40411 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e09398052c91,40411,1731130820128 2024-11-09T05:40:21,576 DEBUG [RS:0;e09398052c91:44035 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e09398052c91,44035,1731130819959' 2024-11-09T05:40:21,576 DEBUG [RS:2;e09398052c91:40411 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e09398052c91,40411,1731130820128' 2024-11-09T05:40:21,576 DEBUG [RS:0;e09398052c91:44035 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T05:40:21,576 DEBUG [RS:2;e09398052c91:40411 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T05:40:21,577 DEBUG [RS:2;e09398052c91:40411 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T05:40:21,577 DEBUG [RS:0;e09398052c91:44035 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T05:40:21,578 DEBUG [RS:2;e09398052c91:40411 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T05:40:21,578 DEBUG [RS:0;e09398052c91:44035 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T05:40:21,578 DEBUG [RS:2;e09398052c91:40411 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T05:40:21,578 DEBUG [RS:0;e09398052c91:44035 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T05:40:21,578 DEBUG [RS:2;e09398052c91:40411 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e09398052c91,40411,1731130820128 2024-11-09T05:40:21,578 DEBUG [RS:0;e09398052c91:44035 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e09398052c91,44035,1731130819959 2024-11-09T05:40:21,578 DEBUG [RS:2;e09398052c91:40411 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e09398052c91,40411,1731130820128' 2024-11-09T05:40:21,578 DEBUG [RS:0;e09398052c91:44035 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e09398052c91,44035,1731130819959' 2024-11-09T05:40:21,578 DEBUG [RS:2;e09398052c91:40411 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T05:40:21,578 DEBUG [RS:0;e09398052c91:44035 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T05:40:21,579 DEBUG [RS:2;e09398052c91:40411 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T05:40:21,579 DEBUG [RS:0;e09398052c91:44035 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T05:40:21,579 DEBUG [RS:0;e09398052c91:44035 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T05:40:21,580 INFO [RS:0;e09398052c91:44035 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T05:40:21,580 INFO [RS:0;e09398052c91:44035 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T05:40:21,580 DEBUG [RS:2;e09398052c91:40411 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T05:40:21,580 INFO [RS:2;e09398052c91:40411 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T05:40:21,580 INFO [RS:2;e09398052c91:40411 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T05:40:21,583 INFO [RS:1;e09398052c91:34331 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:21,583 INFO [RS:1;e09398052c91:34331 {}] regionserver.HRegionServer(1482): Serving as e09398052c91,34331,1731130820076, RpcServer on e09398052c91/172.17.0.2:34331, sessionid=0x100fb3de9450002 2024-11-09T05:40:21,583 DEBUG [RS:1;e09398052c91:34331 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T05:40:21,583 DEBUG [RS:1;e09398052c91:34331 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e09398052c91,34331,1731130820076 2024-11-09T05:40:21,583 DEBUG [RS:1;e09398052c91:34331 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e09398052c91,34331,1731130820076' 2024-11-09T05:40:21,583 DEBUG [RS:1;e09398052c91:34331 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T05:40:21,584 DEBUG [RS:1;e09398052c91:34331 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T05:40:21,585 DEBUG [RS:1;e09398052c91:34331 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T05:40:21,585 DEBUG [RS:1;e09398052c91:34331 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T05:40:21,585 DEBUG [RS:1;e09398052c91:34331 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e09398052c91,34331,1731130820076 2024-11-09T05:40:21,585 DEBUG [RS:1;e09398052c91:34331 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e09398052c91,34331,1731130820076' 2024-11-09T05:40:21,585 DEBUG [RS:1;e09398052c91:34331 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T05:40:21,585 DEBUG [RS:1;e09398052c91:34331 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T05:40:21,586 DEBUG [RS:1;e09398052c91:34331 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T05:40:21,586 INFO [RS:1;e09398052c91:34331 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T05:40:21,586 INFO [RS:1;e09398052c91:34331 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T05:40:21,588 WARN [e09398052c91:34809 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T05:40:21,685 INFO [RS:2;e09398052c91:40411 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T05:40:21,685 INFO [RS:0;e09398052c91:44035 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T05:40:21,687 INFO [RS:1;e09398052c91:34331 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T05:40:21,689 INFO [RS:0;e09398052c91:44035 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e09398052c91%2C44035%2C1731130819959, suffix=, logDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,44035,1731130819959, archiveDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/oldWALs, maxLogs=32 2024-11-09T05:40:21,689 INFO [RS:2;e09398052c91:40411 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e09398052c91%2C40411%2C1731130820128, suffix=, logDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,40411,1731130820128, archiveDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/oldWALs, maxLogs=32 2024-11-09T05:40:21,689 INFO [RS:1;e09398052c91:34331 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e09398052c91%2C34331%2C1731130820076, suffix=, logDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076, archiveDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/oldWALs, maxLogs=32 2024-11-09T05:40:21,707 DEBUG [RS:1;e09398052c91:34331 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076/e09398052c91%2C34331%2C1731130820076.1731130821693, exclude list is [], retry=0 2024-11-09T05:40:21,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43869,DS-ed00c975-41ab-41f9-90d6-754aff306015,DISK] 2024-11-09T05:40:21,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34201,DS-8282eccd-3379-4ce2-9612-3e9a0257084e,DISK] 2024-11-09T05:40:21,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39285,DS-0dfa7937-2ca7-4bc6-8d88-b6076a7fadac,DISK] 2024-11-09T05:40:21,733 DEBUG [RS:0;e09398052c91:44035 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,44035,1731130819959/e09398052c91%2C44035%2C1731130819959.1731130821693, exclude list is [], retry=0 2024-11-09T05:40:21,733 DEBUG [RS:2;e09398052c91:40411 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,40411,1731130820128/e09398052c91%2C40411%2C1731130820128.1731130821693, exclude list is [], retry=0 2024-11-09T05:40:21,735 WARN [IPC Server handler 0 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:21,735 INFO [RS:1;e09398052c91:34331 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076/e09398052c91%2C34331%2C1731130820076.1731130821693 2024-11-09T05:40:21,735 WARN [IPC Server handler 0 on default port 43215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T05:40:21,736 WARN [IPC Server handler 0 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T05:40:21,736 DEBUG [RS:1;e09398052c91:34331 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46203:46203),(127.0.0.1/127.0.0.1:44645:44645),(127.0.0.1/127.0.0.1:42387:42387)] 2024-11-09T05:40:21,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39285,DS-0dfa7937-2ca7-4bc6-8d88-b6076a7fadac,DISK] 2024-11-09T05:40:21,739 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39285,DS-0dfa7937-2ca7-4bc6-8d88-b6076a7fadac,DISK] 2024-11-09T05:40:21,739 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34201,DS-8282eccd-3379-4ce2-9612-3e9a0257084e,DISK] 2024-11-09T05:40:21,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43869,DS-ed00c975-41ab-41f9-90d6-754aff306015,DISK] 2024-11-09T05:40:21,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43869,DS-ed00c975-41ab-41f9-90d6-754aff306015,DISK] 2024-11-09T05:40:21,749 INFO [RS:0;e09398052c91:44035 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,44035,1731130819959/e09398052c91%2C44035%2C1731130819959.1731130821693 2024-11-09T05:40:21,752 DEBUG [RS:0;e09398052c91:44035 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46203:46203),(127.0.0.1/127.0.0.1:42387:42387),(127.0.0.1/127.0.0.1:44645:44645)] 2024-11-09T05:40:21,754 INFO [RS:2;e09398052c91:40411 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,40411,1731130820128/e09398052c91%2C40411%2C1731130820128.1731130821693 2024-11-09T05:40:21,754 DEBUG [RS:2;e09398052c91:40411 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42387:42387),(127.0.0.1/127.0.0.1:44645:44645)] 2024-11-09T05:40:21,840 DEBUG [e09398052c91:34809 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-09T05:40:21,848 DEBUG [e09398052c91:34809 {}] balancer.BalancerClusterState(204): Hosts are {e09398052c91=0} racks are {/default-rack=0} 2024-11-09T05:40:21,854 DEBUG [e09398052c91:34809 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-09T05:40:21,854 DEBUG [e09398052c91:34809 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-09T05:40:21,854 DEBUG [e09398052c91:34809 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-09T05:40:21,854 DEBUG [e09398052c91:34809 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-09T05:40:21,854 DEBUG [e09398052c91:34809 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-09T05:40:21,855 DEBUG [e09398052c91:34809 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-09T05:40:21,855 INFO [e09398052c91:34809 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-09T05:40:21,855 INFO [e09398052c91:34809 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-09T05:40:21,855 INFO [e09398052c91:34809 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-09T05:40:21,855 DEBUG [e09398052c91:34809 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-09T05:40:21,862 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e09398052c91,34331,1731130820076 2024-11-09T05:40:21,868 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,34331,1731130820076, state=OPENING 2024-11-09T05:40:21,873 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-09T05:40:21,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:21,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34331-0x100fb3de9450002, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:21,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:21,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:21,876 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:21,876 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:21,876 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:21,876 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:21,877 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:21,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,34331,1731130820076}] 2024-11-09T05:40:22,053 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T05:40:22,056 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40155, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T05:40:22,068 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-09T05:40:22,068 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T05:40:22,069 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-09T05:40:22,072 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e09398052c91%2C34331%2C1731130820076.meta, suffix=.meta, logDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076, archiveDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/oldWALs, maxLogs=32 2024-11-09T05:40:22,089 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076/e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, exclude list is [], retry=0 2024-11-09T05:40:22,091 WARN [IPC Server handler 2 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:22,091 WARN [IPC Server handler 2 on default port 43215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T05:40:22,091 WARN [IPC Server handler 2 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T05:40:22,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43869,DS-ed00c975-41ab-41f9-90d6-754aff306015,DISK] 2024-11-09T05:40:22,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39285,DS-0dfa7937-2ca7-4bc6-8d88-b6076a7fadac,DISK] 2024-11-09T05:40:22,096 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076/e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta 2024-11-09T05:40:22,097 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44645:44645),(127.0.0.1/127.0.0.1:42387:42387)] 2024-11-09T05:40:22,097 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-09T05:40:22,098 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-09T05:40:22,101 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-09T05:40:22,107 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-09T05:40:22,113 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-09T05:40:22,114 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,114 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-09T05:40:22,114 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-09T05:40:22,118 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T05:40:22,120 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T05:40:22,120 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:22,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T05:40:22,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T05:40:22,123 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T05:40:22,123 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:22,124 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T05:40:22,124 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T05:40:22,125 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T05:40:22,125 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:22,126 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T05:40:22,126 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T05:40:22,128 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T05:40:22,128 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:22,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T05:40:22,129 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T05:40:22,130 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740 2024-11-09T05:40:22,133 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740 2024-11-09T05:40:22,136 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T05:40:22,136 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T05:40:22,137 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T05:40:22,140 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T05:40:22,142 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70669584, jitterRate=0.05305886268615723}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T05:40:22,142 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-09T05:40:22,144 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731130822115Writing region info on filesystem at 1731130822115Initializing all the Stores at 1731130822117 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731130822117Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731130822118 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130822118Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731130822118Cleaning up temporary data from old regions at 1731130822136 (+18 ms)Running coprocessor post-open hooks at 1731130822142 (+6 ms)Region opened successfully at 1731130822144 (+2 ms) 2024-11-09T05:40:22,153 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731130822044 2024-11-09T05:40:22,165 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-09T05:40:22,165 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-09T05:40:22,166 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,34331,1731130820076 2024-11-09T05:40:22,168 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,34331,1731130820076, state=OPEN 2024-11-09T05:40:22,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:22,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:22,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:22,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34331-0x100fb3de9450002, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:22,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:22,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:22,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:22,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:22,172 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e09398052c91,34331,1731130820076 2024-11-09T05:40:22,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-09T05:40:22,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,34331,1731130820076 in 293 msec 2024-11-09T05:40:22,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-09T05:40:22,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 757 msec 2024-11-09T05:40:22,187 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:22,187 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-09T05:40:22,208 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T05:40:22,209 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e09398052c91,34331,1731130820076, seqNum=-1] 2024-11-09T05:40:22,230 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T05:40:22,233 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39985, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T05:40:22,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0700 sec 2024-11-09T05:40:22,255 INFO [master/e09398052c91:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731130822255, completionTime=-1 2024-11-09T05:40:22,258 INFO [master/e09398052c91:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-09T05:40:22,258 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:22,286 INFO [master/e09398052c91:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-09T05:40:22,286 INFO [master/e09398052c91:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130882286 2024-11-09T05:40:22,287 INFO [master/e09398052c91:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130942287 2024-11-09T05:40:22,287 INFO [master/e09398052c91:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-11-09T05:40:22,288 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:22,295 INFO [master/e09398052c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e09398052c91,34809,1731130819107-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:22,295 INFO [master/e09398052c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e09398052c91,34809,1731130819107-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:22,296 INFO [master/e09398052c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e09398052c91,34809,1731130819107-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:22,297 INFO [master/e09398052c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e09398052c91:34809, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:22,297 INFO [master/e09398052c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:22,298 INFO [master/e09398052c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:22,304 DEBUG [master/e09398052c91:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-09T05:40:22,325 INFO [master/e09398052c91:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.089sec 2024-11-09T05:40:22,326 INFO [master/e09398052c91:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-09T05:40:22,327 INFO [master/e09398052c91:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-09T05:40:22,328 INFO [master/e09398052c91:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-09T05:40:22,329 INFO [master/e09398052c91:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-09T05:40:22,329 INFO [master/e09398052c91:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-09T05:40:22,330 INFO [master/e09398052c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e09398052c91,34809,1731130819107-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T05:40:22,330 INFO [master/e09398052c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e09398052c91,34809,1731130819107-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-09T05:40:22,334 DEBUG [master/e09398052c91:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-09T05:40:22,335 INFO [master/e09398052c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-09T05:40:22,336 INFO [master/e09398052c91:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e09398052c91,34809,1731130819107-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T05:40:22,385 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d825df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T05:40:22,389 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-09T05:40:22,389 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-09T05:40:22,393 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e09398052c91,34809,-1 for getting cluster id 2024-11-09T05:40:22,396 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T05:40:22,404 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'eb26d979-02fa-4bab-82e5-c5ab4630e8d4' 2024-11-09T05:40:22,406 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T05:40:22,406 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "eb26d979-02fa-4bab-82e5-c5ab4630e8d4" 2024-11-09T05:40:22,407 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d55d321, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T05:40:22,407 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e09398052c91,34809,-1] 2024-11-09T05:40:22,409 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T05:40:22,411 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T05:40:22,412 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T05:40:22,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5164b956, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T05:40:22,416 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T05:40:22,424 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e09398052c91,34331,1731130820076, seqNum=-1] 2024-11-09T05:40:22,424 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T05:40:22,427 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T05:40:22,448 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e09398052c91,34809,1731130819107 2024-11-09T05:40:22,452 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T05:40:22,457 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e09398052c91,34809,1731130819107 2024-11-09T05:40:22,461 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4b2948c2 2024-11-09T05:40:22,462 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T05:40:22,464 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T05:40:22,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T05:40:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:22,482 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T05:40:22,484 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-09T05:40:22,485 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:22,488 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T05:40:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T05:40:22,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741837_1013 (size=358) 2024-11-09T05:40:22,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741837_1013 (size=358) 2024-11-09T05:40:22,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741837_1013 (size=358) 2024-11-09T05:40:22,509 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 42ac5695b6e0e52a5e46661734af96cc, NAME => 'test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,509 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 9fee264e99db7cad2e2fb211d7666c11, NAME => 'test,bbb,1731130822466.9fee264e99db7cad2e2fb211d7666c11.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,509 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 056a4a6d72fcadc66bad84500d464f5b, NAME => 'test,,1731130822466.056a4a6d72fcadc66bad84500d464f5b.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,514 WARN [IPC Server handler 2 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:22,514 WARN [IPC Server handler 2 on default port 43215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T05:40:22,514 WARN [IPC Server handler 2 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T05:40:22,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741840_1016 (size=42) 2024-11-09T05:40:22,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741840_1016 (size=42) 2024-11-09T05:40:22,560 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1731130822466.056a4a6d72fcadc66bad84500d464f5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741839_1015 (size=45) 2024-11-09T05:40:22,561 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 056a4a6d72fcadc66bad84500d464f5b, disabling compactions & flushes 2024-11-09T05:40:22,561 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1731130822466.056a4a6d72fcadc66bad84500d464f5b. 2024-11-09T05:40:22,561 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731130822466.056a4a6d72fcadc66bad84500d464f5b. 2024-11-09T05:40:22,561 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1731130822466.056a4a6d72fcadc66bad84500d464f5b. after waiting 0 ms 2024-11-09T05:40:22,561 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1731130822466.056a4a6d72fcadc66bad84500d464f5b. 2024-11-09T05:40:22,561 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1731130822466.056a4a6d72fcadc66bad84500d464f5b. 2024-11-09T05:40:22,561 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 056a4a6d72fcadc66bad84500d464f5b: Waiting for close lock at 1731130822560Disabling compacts and flushes for region at 1731130822560Disabling writes for close at 1731130822561 (+1 ms)Writing region close event to WAL at 1731130822561Closed at 1731130822561 2024-11-09T05:40:22,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741839_1015 (size=45) 2024-11-09T05:40:22,562 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 51844aa86315c53347f864258b9b5926, NAME => 'test,ddd,1731130822466.51844aa86315c53347f864258b9b5926.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741839_1015 (size=45) 2024-11-09T05:40:22,566 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:22,566 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:22,566 WARN [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T05:40:22,566 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T05:40:22,567 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:22,567 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:22,567 WARN [RedundancyMonitor {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK, ARCHIVE], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T05:40:22,567 WARN [RedundancyMonitor {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=false) All required storage types are unavailable: unavailableStorages=[DISK, ARCHIVE], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T05:40:22,568 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,568 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 42ac5695b6e0e52a5e46661734af96cc, disabling compactions & flushes 2024-11-09T05:40:22,568 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc. 2024-11-09T05:40:22,568 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc. 2024-11-09T05:40:22,568 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc. after waiting 0 ms 2024-11-09T05:40:22,568 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc. 2024-11-09T05:40:22,568 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc. 2024-11-09T05:40:22,568 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 42ac5695b6e0e52a5e46661734af96cc: Waiting for close lock at 1731130822568Disabling compacts and flushes for region at 1731130822568Disabling writes for close at 1731130822568Writing region close event to WAL at 1731130822568Closed at 1731130822568 2024-11-09T05:40:22,569 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => b4485a6989fa449d3718b74dd5bef8f3, NAME => 'test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,578 WARN [IPC Server handler 4 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:22,578 WARN [IPC Server handler 4 on default port 43215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T05:40:22,578 WARN [IPC Server handler 4 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T05:40:22,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741838_1014 (size=45) 2024-11-09T05:40:22,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741838_1014 (size=45) 2024-11-09T05:40:22,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741838_1014 (size=45) 2024-11-09T05:40:22,581 WARN [IPC Server handler 3 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:22,581 WARN [IPC Server handler 3 on default port 43215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T05:40:22,581 WARN [IPC Server handler 3 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T05:40:22,583 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1731130822466.9fee264e99db7cad2e2fb211d7666c11.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,583 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 9fee264e99db7cad2e2fb211d7666c11, disabling compactions & flushes 2024-11-09T05:40:22,583 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1731130822466.9fee264e99db7cad2e2fb211d7666c11. 2024-11-09T05:40:22,583 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731130822466.9fee264e99db7cad2e2fb211d7666c11. 2024-11-09T05:40:22,583 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731130822466.9fee264e99db7cad2e2fb211d7666c11. after waiting 0 ms 2024-11-09T05:40:22,583 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731130822466.9fee264e99db7cad2e2fb211d7666c11. 2024-11-09T05:40:22,583 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1731130822466.9fee264e99db7cad2e2fb211d7666c11. 2024-11-09T05:40:22,583 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 9fee264e99db7cad2e2fb211d7666c11: Waiting for close lock at 1731130822583Disabling compacts and flushes for region at 1731130822583Disabling writes for close at 1731130822583Writing region close event to WAL at 1731130822583Closed at 1731130822583 2024-11-09T05:40:22,584 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 22143fefe98480e8609aa64fb5bbe23e, NAME => 'test,fff,1731130822466.22143fefe98480e8609aa64fb5bbe23e.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741841_1017 (size=45) 2024-11-09T05:40:22,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741841_1017 (size=45) 2024-11-09T05:40:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T05:40:22,600 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ddd,1731130822466.51844aa86315c53347f864258b9b5926.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,601 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 51844aa86315c53347f864258b9b5926, disabling compactions & flushes 2024-11-09T05:40:22,601 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ddd,1731130822466.51844aa86315c53347f864258b9b5926. 2024-11-09T05:40:22,601 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731130822466.51844aa86315c53347f864258b9b5926. 2024-11-09T05:40:22,601 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731130822466.51844aa86315c53347f864258b9b5926. after waiting 0 ms 2024-11-09T05:40:22,601 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731130822466.51844aa86315c53347f864258b9b5926. 2024-11-09T05:40:22,601 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ddd,1731130822466.51844aa86315c53347f864258b9b5926. 2024-11-09T05:40:22,601 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 51844aa86315c53347f864258b9b5926: Waiting for close lock at 1731130822601Disabling compacts and flushes for region at 1731130822601Disabling writes for close at 1731130822601Writing region close event to WAL at 1731130822601Closed at 1731130822601 2024-11-09T05:40:22,602 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => df09fc3517c167e05c71f65345ca6b80, NAME => 'test,ggg,1731130822466.df09fc3517c167e05c71f65345ca6b80.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741842_1018 (size=45) 2024-11-09T05:40:22,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741842_1018 (size=45) 2024-11-09T05:40:22,607 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,607 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing b4485a6989fa449d3718b74dd5bef8f3, disabling compactions & flushes 2024-11-09T05:40:22,607 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3. 2024-11-09T05:40:22,607 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3. 2024-11-09T05:40:22,607 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3. after waiting 0 ms 2024-11-09T05:40:22,607 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3. 2024-11-09T05:40:22,607 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3. 2024-11-09T05:40:22,607 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for b4485a6989fa449d3718b74dd5bef8f3: Waiting for close lock at 1731130822607Disabling compacts and flushes for region at 1731130822607Disabling writes for close at 1731130822607Writing region close event to WAL at 1731130822607Closed at 1731130822607 2024-11-09T05:40:22,608 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => bd305d310e01738dc5cbabbfddb8eb5c, NAME => 'test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,610 WARN [IPC Server handler 1 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:22,610 WARN [IPC Server handler 1 on default port 43215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T05:40:22,610 WARN [IPC Server handler 1 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T05:40:22,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741843_1019 (size=45) 2024-11-09T05:40:22,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741843_1019 (size=45) 2024-11-09T05:40:22,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741843_1019 (size=45) 2024-11-09T05:40:22,618 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,fff,1731130822466.22143fefe98480e8609aa64fb5bbe23e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,618 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 22143fefe98480e8609aa64fb5bbe23e, disabling compactions & flushes 2024-11-09T05:40:22,618 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,fff,1731130822466.22143fefe98480e8609aa64fb5bbe23e. 2024-11-09T05:40:22,618 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731130822466.22143fefe98480e8609aa64fb5bbe23e. 2024-11-09T05:40:22,618 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731130822466.22143fefe98480e8609aa64fb5bbe23e. after waiting 0 ms 2024-11-09T05:40:22,618 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731130822466.22143fefe98480e8609aa64fb5bbe23e. 2024-11-09T05:40:22,619 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,fff,1731130822466.22143fefe98480e8609aa64fb5bbe23e. 2024-11-09T05:40:22,619 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 22143fefe98480e8609aa64fb5bbe23e: Waiting for close lock at 1731130822618Disabling compacts and flushes for region at 1731130822618Disabling writes for close at 1731130822618Writing region close event to WAL at 1731130822618Closed at 1731130822618 2024-11-09T05:40:22,619 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 348c820a04c95623395fb88f37fad187, NAME => 'test,iii,1731130822466.348c820a04c95623395fb88f37fad187.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741845_1021 (size=45) 2024-11-09T05:40:22,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741844_1020 (size=45) 2024-11-09T05:40:22,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741844_1020 (size=45) 2024-11-09T05:40:22,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741845_1021 (size=45) 2024-11-09T05:40:22,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741845_1021 (size=45) 2024-11-09T05:40:22,624 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ggg,1731130822466.df09fc3517c167e05c71f65345ca6b80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,624 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing df09fc3517c167e05c71f65345ca6b80, disabling compactions & flushes 2024-11-09T05:40:22,624 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ggg,1731130822466.df09fc3517c167e05c71f65345ca6b80. 2024-11-09T05:40:22,624 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731130822466.df09fc3517c167e05c71f65345ca6b80. 2024-11-09T05:40:22,624 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731130822466.df09fc3517c167e05c71f65345ca6b80. after waiting 0 ms 2024-11-09T05:40:22,624 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731130822466.df09fc3517c167e05c71f65345ca6b80. 2024-11-09T05:40:22,624 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ggg,1731130822466.df09fc3517c167e05c71f65345ca6b80. 2024-11-09T05:40:22,624 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for df09fc3517c167e05c71f65345ca6b80: Waiting for close lock at 1731130822624Disabling compacts and flushes for region at 1731130822624Disabling writes for close at 1731130822624Writing region close event to WAL at 1731130822624Closed at 1731130822624 2024-11-09T05:40:22,624 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,625 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing bd305d310e01738dc5cbabbfddb8eb5c, disabling compactions & flushes 2024-11-09T05:40:22,625 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6ea3613715cd8a321124b73881db15ce, NAME => 'test,jjj,1731130822466.6ea3613715cd8a321124b73881db15ce.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,625 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c. 2024-11-09T05:40:22,625 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c. 2024-11-09T05:40:22,625 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c. after waiting 0 ms 2024-11-09T05:40:22,625 WARN [IPC Server handler 2 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:22,625 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c. 2024-11-09T05:40:22,625 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c. 2024-11-09T05:40:22,625 WARN [IPC Server handler 2 on default port 43215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T05:40:22,625 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for bd305d310e01738dc5cbabbfddb8eb5c: Waiting for close lock at 1731130822625Disabling compacts and flushes for region at 1731130822625Disabling writes for close at 1731130822625Writing region close event to WAL at 1731130822625Closed at 1731130822625 2024-11-09T05:40:22,625 WARN [IPC Server handler 2 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T05:40:22,626 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 706d9970ba371fdec8f2a079940ca3ae, NAME => 'test,kkk,1731130822466.706d9970ba371fdec8f2a079940ca3ae.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,630 WARN [IPC Server handler 3 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:22,630 WARN [IPC Server handler 3 on default port 43215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T05:40:22,630 WARN [IPC Server handler 3 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T05:40:22,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741846_1022 (size=45) 2024-11-09T05:40:22,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741846_1022 (size=45) 2024-11-09T05:40:22,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741848_1024 (size=45) 2024-11-09T05:40:22,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741848_1024 (size=45) 2024-11-09T05:40:22,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741847_1023 (size=45) 2024-11-09T05:40:22,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741847_1023 (size=45) 2024-11-09T05:40:22,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741847_1023 (size=45) 2024-11-09T05:40:22,641 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,jjj,1731130822466.6ea3613715cd8a321124b73881db15ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,641 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,kkk,1731130822466.706d9970ba371fdec8f2a079940ca3ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,641 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 6ea3613715cd8a321124b73881db15ce, disabling compactions & flushes 2024-11-09T05:40:22,641 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 706d9970ba371fdec8f2a079940ca3ae, disabling compactions & flushes 2024-11-09T05:40:22,641 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,jjj,1731130822466.6ea3613715cd8a321124b73881db15ce. 2024-11-09T05:40:22,641 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731130822466.6ea3613715cd8a321124b73881db15ce. 2024-11-09T05:40:22,641 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,kkk,1731130822466.706d9970ba371fdec8f2a079940ca3ae. 2024-11-09T05:40:22,641 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731130822466.6ea3613715cd8a321124b73881db15ce. after waiting 0 ms 2024-11-09T05:40:22,641 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731130822466.706d9970ba371fdec8f2a079940ca3ae. 2024-11-09T05:40:22,642 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731130822466.6ea3613715cd8a321124b73881db15ce. 2024-11-09T05:40:22,642 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731130822466.706d9970ba371fdec8f2a079940ca3ae. after waiting 0 ms 2024-11-09T05:40:22,642 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,jjj,1731130822466.6ea3613715cd8a321124b73881db15ce. 2024-11-09T05:40:22,642 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731130822466.706d9970ba371fdec8f2a079940ca3ae. 2024-11-09T05:40:22,642 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6ea3613715cd8a321124b73881db15ce: Waiting for close lock at 1731130822641Disabling compacts and flushes for region at 1731130822641Disabling writes for close at 1731130822642 (+1 ms)Writing region close event to WAL at 1731130822642Closed at 1731130822642 2024-11-09T05:40:22,642 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,kkk,1731130822466.706d9970ba371fdec8f2a079940ca3ae. 2024-11-09T05:40:22,642 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 706d9970ba371fdec8f2a079940ca3ae: Waiting for close lock at 1731130822641Disabling compacts and flushes for region at 1731130822641Disabling writes for close at 1731130822642 (+1 ms)Writing region close event to WAL at 1731130822642Closed at 1731130822642 2024-11-09T05:40:22,642 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 12126c11ff675bc823b55beab9656286, NAME => 'test,lll,1731130822466.12126c11ff675bc823b55beab9656286.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,643 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 6c8538a5c08a601164d6d90f17409b33, NAME => 'test,mmm,1731130822466.6c8538a5c08a601164d6d90f17409b33.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,647 WARN [IPC Server handler 3 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:22,647 WARN [IPC Server handler 1 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:22,647 WARN [IPC Server handler 3 on default port 43215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T05:40:22,647 WARN [IPC Server handler 1 on default port 43215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T05:40:22,647 WARN [IPC Server handler 3 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T05:40:22,647 WARN [IPC Server handler 1 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T05:40:22,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741850_1026 (size=45) 2024-11-09T05:40:22,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741850_1026 (size=45) 2024-11-09T05:40:22,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741849_1025 (size=45) 2024-11-09T05:40:22,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741849_1025 (size=45) 2024-11-09T05:40:22,655 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,lll,1731130822466.12126c11ff675bc823b55beab9656286.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,655 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 12126c11ff675bc823b55beab9656286, disabling compactions & flushes 2024-11-09T05:40:22,655 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,lll,1731130822466.12126c11ff675bc823b55beab9656286. 2024-11-09T05:40:22,655 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731130822466.12126c11ff675bc823b55beab9656286. 2024-11-09T05:40:22,655 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731130822466.12126c11ff675bc823b55beab9656286. after waiting 0 ms 2024-11-09T05:40:22,655 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731130822466.12126c11ff675bc823b55beab9656286. 2024-11-09T05:40:22,655 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,lll,1731130822466.12126c11ff675bc823b55beab9656286. 2024-11-09T05:40:22,656 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,mmm,1731130822466.6c8538a5c08a601164d6d90f17409b33.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,656 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 12126c11ff675bc823b55beab9656286: Waiting for close lock at 1731130822655Disabling compacts and flushes for region at 1731130822655Disabling writes for close at 1731130822655Writing region close event to WAL at 1731130822655Closed at 1731130822655 2024-11-09T05:40:22,656 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 6c8538a5c08a601164d6d90f17409b33, disabling compactions & flushes 2024-11-09T05:40:22,656 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,mmm,1731130822466.6c8538a5c08a601164d6d90f17409b33. 2024-11-09T05:40:22,656 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731130822466.6c8538a5c08a601164d6d90f17409b33. 2024-11-09T05:40:22,656 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731130822466.6c8538a5c08a601164d6d90f17409b33. after waiting 0 ms 2024-11-09T05:40:22,656 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731130822466.6c8538a5c08a601164d6d90f17409b33. 2024-11-09T05:40:22,656 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,mmm,1731130822466.6c8538a5c08a601164d6d90f17409b33. 2024-11-09T05:40:22,656 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 6c8538a5c08a601164d6d90f17409b33: Waiting for close lock at 1731130822656Disabling compacts and flushes for region at 1731130822656Disabling writes for close at 1731130822656Writing region close event to WAL at 1731130822656Closed at 1731130822656 2024-11-09T05:40:22,656 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ce006577ade7d81e0bf87827254771f5, NAME => 'test,nnn,1731130822466.ce006577ade7d81e0bf87827254771f5.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,657 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 8d63f643dc4acf4d7c54bcb479b29df0, NAME => 'test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,661 WARN [IPC Server handler 3 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:22,661 WARN [IPC Server handler 3 on default port 43215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T05:40:22,661 WARN [IPC Server handler 3 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T05:40:22,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741852_1028 (size=45) 2024-11-09T05:40:22,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741851_1027 (size=45) 2024-11-09T05:40:22,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741851_1027 (size=45) 2024-11-09T05:40:22,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741851_1027 (size=45) 2024-11-09T05:40:22,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741852_1028 (size=45) 2024-11-09T05:40:22,673 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1731130822466.ce006577ade7d81e0bf87827254771f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,673 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,673 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing ce006577ade7d81e0bf87827254771f5, disabling compactions & flushes 2024-11-09T05:40:22,673 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1731130822466.ce006577ade7d81e0bf87827254771f5. 2024-11-09T05:40:22,673 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731130822466.ce006577ade7d81e0bf87827254771f5. 2024-11-09T05:40:22,673 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 8d63f643dc4acf4d7c54bcb479b29df0, disabling compactions & flushes 2024-11-09T05:40:22,673 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731130822466.ce006577ade7d81e0bf87827254771f5. after waiting 0 ms 2024-11-09T05:40:22,673 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0. 2024-11-09T05:40:22,673 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731130822466.ce006577ade7d81e0bf87827254771f5. 2024-11-09T05:40:22,673 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1731130822466.ce006577ade7d81e0bf87827254771f5. 2024-11-09T05:40:22,673 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0. 2024-11-09T05:40:22,674 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for ce006577ade7d81e0bf87827254771f5: Waiting for close lock at 1731130822673Disabling compacts and flushes for region at 1731130822673Disabling writes for close at 1731130822673Writing region close event to WAL at 1731130822673Closed at 1731130822673 2024-11-09T05:40:22,674 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0. after waiting 0 ms 2024-11-09T05:40:22,674 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0. 2024-11-09T05:40:22,674 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0. 2024-11-09T05:40:22,674 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 8d63f643dc4acf4d7c54bcb479b29df0: Waiting for close lock at 1731130822673Disabling compacts and flushes for region at 1731130822673Disabling writes for close at 1731130822674 (+1 ms)Writing region close event to WAL at 1731130822674Closed at 1731130822674 2024-11-09T05:40:22,674 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d014bc791a8473500fbbfa044123f955, NAME => 'test,ppp,1731130822466.d014bc791a8473500fbbfa044123f955.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,675 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 8d8a7929b729c2e22434e4a1d52e461a, NAME => 'test,qqq,1731130822466.8d8a7929b729c2e22434e4a1d52e461a.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,679 WARN [IPC Server handler 2 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:22,679 WARN [IPC Server handler 0 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:22,679 WARN [IPC Server handler 2 on default port 43215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T05:40:22,679 WARN [IPC Server handler 2 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T05:40:22,679 WARN [IPC Server handler 0 on default port 43215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T05:40:22,679 WARN [IPC Server handler 0 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T05:40:22,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741854_1030 (size=45) 2024-11-09T05:40:22,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741854_1030 (size=45) 2024-11-09T05:40:22,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741853_1029 (size=45) 2024-11-09T05:40:22,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741853_1029 (size=45) 2024-11-09T05:40:22,686 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ppp,1731130822466.d014bc791a8473500fbbfa044123f955.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,686 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing d014bc791a8473500fbbfa044123f955, disabling compactions & flushes 2024-11-09T05:40:22,686 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ppp,1731130822466.d014bc791a8473500fbbfa044123f955. 2024-11-09T05:40:22,686 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731130822466.d014bc791a8473500fbbfa044123f955. 2024-11-09T05:40:22,686 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,qqq,1731130822466.8d8a7929b729c2e22434e4a1d52e461a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,686 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731130822466.d014bc791a8473500fbbfa044123f955. after waiting 0 ms 2024-11-09T05:40:22,686 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731130822466.d014bc791a8473500fbbfa044123f955. 2024-11-09T05:40:22,686 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ppp,1731130822466.d014bc791a8473500fbbfa044123f955. 2024-11-09T05:40:22,686 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for d014bc791a8473500fbbfa044123f955: Waiting for close lock at 1731130822686Disabling compacts and flushes for region at 1731130822686Disabling writes for close at 1731130822686Writing region close event to WAL at 1731130822686Closed at 1731130822686 2024-11-09T05:40:22,686 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 8d8a7929b729c2e22434e4a1d52e461a, disabling compactions & flushes 2024-11-09T05:40:22,687 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,qqq,1731130822466.8d8a7929b729c2e22434e4a1d52e461a. 2024-11-09T05:40:22,687 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8714188f3856e307485ef7426df7ea3c, NAME => 'test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,687 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731130822466.8d8a7929b729c2e22434e4a1d52e461a. 2024-11-09T05:40:22,687 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731130822466.8d8a7929b729c2e22434e4a1d52e461a. after waiting 0 ms 2024-11-09T05:40:22,687 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731130822466.8d8a7929b729c2e22434e4a1d52e461a. 2024-11-09T05:40:22,687 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,qqq,1731130822466.8d8a7929b729c2e22434e4a1d52e461a. 2024-11-09T05:40:22,687 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 8d8a7929b729c2e22434e4a1d52e461a: Waiting for close lock at 1731130822686Disabling compacts and flushes for region at 1731130822686Disabling writes for close at 1731130822687 (+1 ms)Writing region close event to WAL at 1731130822687Closed at 1731130822687 2024-11-09T05:40:22,688 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => c06857dd8e828d035329855bf73ed41f, NAME => 'test,sss,1731130822466.c06857dd8e828d035329855bf73ed41f.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,691 WARN [IPC Server handler 2 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:22,691 WARN [IPC Server handler 2 on default port 43215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T05:40:22,691 WARN [IPC Server handler 2 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T05:40:22,692 WARN [IPC Server handler 4 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:22,692 WARN [IPC Server handler 4 on default port 43215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T05:40:22,692 WARN [IPC Server handler 4 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T05:40:22,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741855_1031 (size=45) 2024-11-09T05:40:22,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741855_1031 (size=45) 2024-11-09T05:40:22,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741856_1032 (size=45) 2024-11-09T05:40:22,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741856_1032 (size=45) 2024-11-09T05:40:22,699 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,699 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 8714188f3856e307485ef7426df7ea3c, disabling compactions & flushes 2024-11-09T05:40:22,699 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c. 2024-11-09T05:40:22,699 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c. 2024-11-09T05:40:22,699 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c. after waiting 0 ms 2024-11-09T05:40:22,700 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c. 2024-11-09T05:40:22,700 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c. 2024-11-09T05:40:22,700 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8714188f3856e307485ef7426df7ea3c: Waiting for close lock at 1731130822699Disabling compacts and flushes for region at 1731130822699Disabling writes for close at 1731130822700 (+1 ms)Writing region close event to WAL at 1731130822700Closed at 1731130822700 2024-11-09T05:40:22,700 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2339918a4a54d7d8274a43da565fe2a2, NAME => 'test,ttt,1731130822466.2339918a4a54d7d8274a43da565fe2a2.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,701 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,sss,1731130822466.c06857dd8e828d035329855bf73ed41f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,701 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing c06857dd8e828d035329855bf73ed41f, disabling compactions & flushes 2024-11-09T05:40:22,701 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,sss,1731130822466.c06857dd8e828d035329855bf73ed41f. 2024-11-09T05:40:22,701 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731130822466.c06857dd8e828d035329855bf73ed41f. 2024-11-09T05:40:22,701 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731130822466.c06857dd8e828d035329855bf73ed41f. after waiting 0 ms 2024-11-09T05:40:22,701 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731130822466.c06857dd8e828d035329855bf73ed41f. 2024-11-09T05:40:22,701 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,sss,1731130822466.c06857dd8e828d035329855bf73ed41f. 2024-11-09T05:40:22,701 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for c06857dd8e828d035329855bf73ed41f: Waiting for close lock at 1731130822701Disabling compacts and flushes for region at 1731130822701Disabling writes for close at 1731130822701Writing region close event to WAL at 1731130822701Closed at 1731130822701 2024-11-09T05:40:22,702 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 532f29d232f4841336f315b0406fe4af, NAME => 'test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,705 WARN [IPC Server handler 0 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:22,705 WARN [IPC Server handler 0 on default port 43215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T05:40:22,705 WARN [IPC Server handler 0 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T05:40:22,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741858_1034 (size=45) 2024-11-09T05:40:22,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741858_1034 (size=45) 2024-11-09T05:40:22,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741857_1033 (size=45) 2024-11-09T05:40:22,712 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ttt,1731130822466.2339918a4a54d7d8274a43da565fe2a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,712 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 2339918a4a54d7d8274a43da565fe2a2, disabling compactions & flushes 2024-11-09T05:40:22,713 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ttt,1731130822466.2339918a4a54d7d8274a43da565fe2a2. 2024-11-09T05:40:22,713 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731130822466.2339918a4a54d7d8274a43da565fe2a2. 2024-11-09T05:40:22,713 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731130822466.2339918a4a54d7d8274a43da565fe2a2. after waiting 0 ms 2024-11-09T05:40:22,713 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731130822466.2339918a4a54d7d8274a43da565fe2a2. 2024-11-09T05:40:22,713 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ttt,1731130822466.2339918a4a54d7d8274a43da565fe2a2. 2024-11-09T05:40:22,713 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2339918a4a54d7d8274a43da565fe2a2: Waiting for close lock at 1731130822712Disabling compacts and flushes for region at 1731130822712Disabling writes for close at 1731130822713 (+1 ms)Writing region close event to WAL at 1731130822713Closed at 1731130822713 2024-11-09T05:40:22,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741857_1033 (size=45) 2024-11-09T05:40:22,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741857_1033 (size=45) 2024-11-09T05:40:22,713 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0f1e5e5fd2628827f76a773887689bc8, NAME => 'test,vvv,1731130822466.0f1e5e5fd2628827f76a773887689bc8.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,715 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,715 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 532f29d232f4841336f315b0406fe4af, disabling compactions & flushes 2024-11-09T05:40:22,715 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af. 2024-11-09T05:40:22,715 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af. 2024-11-09T05:40:22,715 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af. after waiting 0 ms 2024-11-09T05:40:22,715 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af. 2024-11-09T05:40:22,715 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af. 2024-11-09T05:40:22,715 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 532f29d232f4841336f315b0406fe4af: Waiting for close lock at 1731130822715Disabling compacts and flushes for region at 1731130822715Disabling writes for close at 1731130822715Writing region close event to WAL at 1731130822715Closed at 1731130822715 2024-11-09T05:40:22,716 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1cc6d417b0bdc1061e6457e90a27badd, NAME => 'test,www,1731130822466.1cc6d417b0bdc1061e6457e90a27badd.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,719 WARN [IPC Server handler 0 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:22,719 WARN [IPC Server handler 0 on default port 43215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T05:40:22,719 WARN [IPC Server handler 0 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T05:40:22,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741859_1035 (size=45) 2024-11-09T05:40:22,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741860_1036 (size=45) 2024-11-09T05:40:22,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741859_1035 (size=45) 2024-11-09T05:40:22,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741859_1035 (size=45) 2024-11-09T05:40:22,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741860_1036 (size=45) 2024-11-09T05:40:22,728 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,vvv,1731130822466.0f1e5e5fd2628827f76a773887689bc8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,729 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 0f1e5e5fd2628827f76a773887689bc8, disabling compactions & flushes 2024-11-09T05:40:22,729 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,vvv,1731130822466.0f1e5e5fd2628827f76a773887689bc8. 2024-11-09T05:40:22,729 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731130822466.0f1e5e5fd2628827f76a773887689bc8. 2024-11-09T05:40:22,729 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,www,1731130822466.1cc6d417b0bdc1061e6457e90a27badd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,729 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731130822466.0f1e5e5fd2628827f76a773887689bc8. after waiting 0 ms 2024-11-09T05:40:22,729 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731130822466.0f1e5e5fd2628827f76a773887689bc8. 2024-11-09T05:40:22,729 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 1cc6d417b0bdc1061e6457e90a27badd, disabling compactions & flushes 2024-11-09T05:40:22,729 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,vvv,1731130822466.0f1e5e5fd2628827f76a773887689bc8. 2024-11-09T05:40:22,729 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,www,1731130822466.1cc6d417b0bdc1061e6457e90a27badd. 2024-11-09T05:40:22,729 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0f1e5e5fd2628827f76a773887689bc8: Waiting for close lock at 1731130822729Disabling compacts and flushes for region at 1731130822729Disabling writes for close at 1731130822729Writing region close event to WAL at 1731130822729Closed at 1731130822729 2024-11-09T05:40:22,729 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731130822466.1cc6d417b0bdc1061e6457e90a27badd. 2024-11-09T05:40:22,729 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1731130822466.1cc6d417b0bdc1061e6457e90a27badd. after waiting 0 ms 2024-11-09T05:40:22,729 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1731130822466.1cc6d417b0bdc1061e6457e90a27badd. 2024-11-09T05:40:22,729 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,www,1731130822466.1cc6d417b0bdc1061e6457e90a27badd. 2024-11-09T05:40:22,729 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 1cc6d417b0bdc1061e6457e90a27badd: Waiting for close lock at 1731130822729Disabling compacts and flushes for region at 1731130822729Disabling writes for close at 1731130822729Writing region close event to WAL at 1731130822729Closed at 1731130822729 2024-11-09T05:40:22,729 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d923c8e1af5892d9575aa4b3336f28f9, NAME => 'test,xxx,1731130822466.d923c8e1af5892d9575aa4b3336f28f9.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,730 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => b609b56af9f6fc87d9a153736ec0c8d1, NAME => 'test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,733 WARN [IPC Server handler 2 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T05:40:22,733 WARN [IPC Server handler 2 on default port 43215 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T05:40:22,733 WARN [IPC Server handler 2 on default port 43215 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T05:40:22,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741862_1038 (size=45) 2024-11-09T05:40:22,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741862_1038 (size=45) 2024-11-09T05:40:22,739 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,739 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing b609b56af9f6fc87d9a153736ec0c8d1, disabling compactions & flushes 2024-11-09T05:40:22,739 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1. 2024-11-09T05:40:22,739 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1. 2024-11-09T05:40:22,739 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1. after waiting 0 ms 2024-11-09T05:40:22,740 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1. 2024-11-09T05:40:22,740 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1. 2024-11-09T05:40:22,740 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for b609b56af9f6fc87d9a153736ec0c8d1: Waiting for close lock at 1731130822739Disabling compacts and flushes for region at 1731130822739Disabling writes for close at 1731130822739Writing region close event to WAL at 1731130822740 (+1 ms)Closed at 1731130822740 2024-11-09T05:40:22,741 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 6874567d13af4b499b6eb873decadd10, NAME => 'test,zzz,1731130822466.6874567d13af4b499b6eb873decadd10.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352 2024-11-09T05:40:22,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741861_1037 (size=45) 2024-11-09T05:40:22,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741861_1037 (size=45) 2024-11-09T05:40:22,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741861_1037 (size=45) 2024-11-09T05:40:22,745 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,xxx,1731130822466.d923c8e1af5892d9575aa4b3336f28f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,745 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing d923c8e1af5892d9575aa4b3336f28f9, disabling compactions & flushes 2024-11-09T05:40:22,745 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,xxx,1731130822466.d923c8e1af5892d9575aa4b3336f28f9. 2024-11-09T05:40:22,745 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731130822466.d923c8e1af5892d9575aa4b3336f28f9. 2024-11-09T05:40:22,745 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731130822466.d923c8e1af5892d9575aa4b3336f28f9. after waiting 0 ms 2024-11-09T05:40:22,745 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731130822466.d923c8e1af5892d9575aa4b3336f28f9. 2024-11-09T05:40:22,745 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,xxx,1731130822466.d923c8e1af5892d9575aa4b3336f28f9. 2024-11-09T05:40:22,745 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for d923c8e1af5892d9575aa4b3336f28f9: Waiting for close lock at 1731130822745Disabling compacts and flushes for region at 1731130822745Disabling writes for close at 1731130822745Writing region close event to WAL at 1731130822745Closed at 1731130822745 2024-11-09T05:40:22,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741863_1039 (size=42) 2024-11-09T05:40:22,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741863_1039 (size=42) 2024-11-09T05:40:22,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741863_1039 (size=42) 2024-11-09T05:40:22,752 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,zzz,1731130822466.6874567d13af4b499b6eb873decadd10.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:22,752 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 6874567d13af4b499b6eb873decadd10, disabling compactions & flushes 2024-11-09T05:40:22,752 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,zzz,1731130822466.6874567d13af4b499b6eb873decadd10. 2024-11-09T05:40:22,752 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731130822466.6874567d13af4b499b6eb873decadd10. 2024-11-09T05:40:22,752 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731130822466.6874567d13af4b499b6eb873decadd10. after waiting 0 ms 2024-11-09T05:40:22,752 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731130822466.6874567d13af4b499b6eb873decadd10. 2024-11-09T05:40:22,752 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,zzz,1731130822466.6874567d13af4b499b6eb873decadd10. 2024-11-09T05:40:22,752 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 6874567d13af4b499b6eb873decadd10: Waiting for close lock at 1731130822752Disabling compacts and flushes for region at 1731130822752Disabling writes for close at 1731130822752Writing region close event to WAL at 1731130822752Closed at 1731130822752 2024-11-09T05:40:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T05:40:23,037 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,iii,1731130822466.348c820a04c95623395fb88f37fad187.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,037 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 348c820a04c95623395fb88f37fad187, disabling compactions & flushes 2024-11-09T05:40:23,037 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,iii,1731130822466.348c820a04c95623395fb88f37fad187. 2024-11-09T05:40:23,037 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731130822466.348c820a04c95623395fb88f37fad187. 2024-11-09T05:40:23,037 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731130822466.348c820a04c95623395fb88f37fad187. after waiting 0 ms 2024-11-09T05:40:23,037 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731130822466.348c820a04c95623395fb88f37fad187. 2024-11-09T05:40:23,037 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,iii,1731130822466.348c820a04c95623395fb88f37fad187. 2024-11-09T05:40:23,037 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 348c820a04c95623395fb88f37fad187: Waiting for close lock at 1731130823037Disabling compacts and flushes for region at 1731130823037Disabling writes for close at 1731130823037Writing region close event to WAL at 1731130823037Closed at 1731130823037 2024-11-09T05:40:23,040 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T05:40:23,045 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1731130822466.056a4a6d72fcadc66bad84500d464f5b.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,045 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,045 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1731130822466.9fee264e99db7cad2e2fb211d7666c11.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,046 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1731130822466.51844aa86315c53347f864258b9b5926.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,046 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,046 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1731130822466.22143fefe98480e8609aa64fb5bbe23e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,046 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1731130822466.df09fc3517c167e05c71f65345ca6b80.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,046 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,046 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1731130822466.6ea3613715cd8a321124b73881db15ce.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,046 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1731130822466.706d9970ba371fdec8f2a079940ca3ae.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,046 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1731130822466.12126c11ff675bc823b55beab9656286.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,047 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1731130822466.6c8538a5c08a601164d6d90f17409b33.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,047 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1731130822466.ce006577ade7d81e0bf87827254771f5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,047 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,047 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1731130822466.d014bc791a8473500fbbfa044123f955.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,047 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1731130822466.8d8a7929b729c2e22434e4a1d52e461a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,047 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,047 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1731130822466.c06857dd8e828d035329855bf73ed41f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,047 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1731130822466.2339918a4a54d7d8274a43da565fe2a2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,047 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,047 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1731130822466.0f1e5e5fd2628827f76a773887689bc8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,047 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1731130822466.1cc6d417b0bdc1061e6457e90a27badd.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,048 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,048 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1731130822466.d923c8e1af5892d9575aa4b3336f28f9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,048 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1731130822466.6874567d13af4b499b6eb873decadd10.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,048 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1731130822466.348c820a04c95623395fb88f37fad187.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731130823040"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731130823040"}]},"ts":"1731130823040"} 2024-11-09T05:40:23,087 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-09T05:40:23,089 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T05:40:23,091 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731130823089"}]},"ts":"1731130823089"} 2024-11-09T05:40:23,096 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-09T05:40:23,097 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {e09398052c91=0} racks are {/default-rack=0} 2024-11-09T05:40:23,113 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-09T05:40:23,113 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-09T05:40:23,113 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-09T05:40:23,113 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-09T05:40:23,113 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-09T05:40:23,113 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-09T05:40:23,113 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-09T05:40:23,113 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-09T05:40:23,114 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-09T05:40:23,114 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-09T05:40:23,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=056a4a6d72fcadc66bad84500d464f5b, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9fee264e99db7cad2e2fb211d7666c11, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=51844aa86315c53347f864258b9b5926, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=22143fefe98480e8609aa64fb5bbe23e, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=df09fc3517c167e05c71f65345ca6b80, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6ea3613715cd8a321124b73881db15ce, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=706d9970ba371fdec8f2a079940ca3ae, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6c8538a5c08a601164d6d90f17409b33, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ce006577ade7d81e0bf87827254771f5, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d014bc791a8473500fbbfa044123f955, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8d8a7929b729c2e22434e4a1d52e461a, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c06857dd8e828d035329855bf73ed41f, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2339918a4a54d7d8274a43da565fe2a2, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0f1e5e5fd2628827f76a773887689bc8, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1cc6d417b0bdc1061e6457e90a27badd, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d923c8e1af5892d9575aa4b3336f28f9, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6874567d13af4b499b6eb873decadd10, ASSIGN}] 2024-11-09T05:40:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T05:40:23,124 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6874567d13af4b499b6eb873decadd10, ASSIGN 2024-11-09T05:40:23,126 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6874567d13af4b499b6eb873decadd10, ASSIGN; state=OFFLINE, location=e09398052c91,40411,1731130820128; forceNewPlan=false, retain=false 2024-11-09T05:40:23,127 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:23,128 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN; state=OFFLINE, location=e09398052c91,34331,1731130820076; forceNewPlan=false, retain=false 2024-11-09T05:40:23,130 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d923c8e1af5892d9575aa4b3336f28f9, ASSIGN 2024-11-09T05:40:23,131 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d923c8e1af5892d9575aa4b3336f28f9, ASSIGN; state=OFFLINE, location=e09398052c91,44035,1731130819959; forceNewPlan=false, retain=false 2024-11-09T05:40:23,132 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1cc6d417b0bdc1061e6457e90a27badd, ASSIGN 2024-11-09T05:40:23,133 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1cc6d417b0bdc1061e6457e90a27badd, ASSIGN; state=OFFLINE, location=e09398052c91,40411,1731130820128; forceNewPlan=false, retain=false 2024-11-09T05:40:23,135 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0f1e5e5fd2628827f76a773887689bc8, ASSIGN 2024-11-09T05:40:23,136 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0f1e5e5fd2628827f76a773887689bc8, ASSIGN; state=OFFLINE, location=e09398052c91,44035,1731130819959; forceNewPlan=false, retain=false 2024-11-09T05:40:23,137 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:23,138 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN; state=OFFLINE, location=e09398052c91,34331,1731130820076; forceNewPlan=false, retain=false 2024-11-09T05:40:23,140 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2339918a4a54d7d8274a43da565fe2a2, ASSIGN 2024-11-09T05:40:23,141 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2339918a4a54d7d8274a43da565fe2a2, ASSIGN; state=OFFLINE, location=e09398052c91,40411,1731130820128; forceNewPlan=false, retain=false 2024-11-09T05:40:23,142 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c06857dd8e828d035329855bf73ed41f, ASSIGN 2024-11-09T05:40:23,143 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c06857dd8e828d035329855bf73ed41f, ASSIGN; state=OFFLINE, location=e09398052c91,44035,1731130819959; forceNewPlan=false, retain=false 2024-11-09T05:40:23,145 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:23,146 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN; state=OFFLINE, location=e09398052c91,34331,1731130820076; forceNewPlan=false, retain=false 2024-11-09T05:40:23,147 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8d8a7929b729c2e22434e4a1d52e461a, ASSIGN 2024-11-09T05:40:23,148 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8d8a7929b729c2e22434e4a1d52e461a, ASSIGN; state=OFFLINE, location=e09398052c91,40411,1731130820128; forceNewPlan=false, retain=false 2024-11-09T05:40:23,149 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d014bc791a8473500fbbfa044123f955, ASSIGN 2024-11-09T05:40:23,150 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d014bc791a8473500fbbfa044123f955, ASSIGN; state=OFFLINE, location=e09398052c91,44035,1731130819959; forceNewPlan=false, retain=false 2024-11-09T05:40:23,152 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, ASSIGN 2024-11-09T05:40:23,153 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, ASSIGN; state=OFFLINE, location=e09398052c91,34331,1731130820076; forceNewPlan=false, retain=false 2024-11-09T05:40:23,155 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ce006577ade7d81e0bf87827254771f5, ASSIGN 2024-11-09T05:40:23,156 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ce006577ade7d81e0bf87827254771f5, ASSIGN; state=OFFLINE, location=e09398052c91,40411,1731130820128; forceNewPlan=false, retain=false 2024-11-09T05:40:23,158 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6c8538a5c08a601164d6d90f17409b33, ASSIGN 2024-11-09T05:40:23,159 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6c8538a5c08a601164d6d90f17409b33, ASSIGN; state=OFFLINE, location=e09398052c91,44035,1731130819959; forceNewPlan=false, retain=false 2024-11-09T05:40:23,161 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN 2024-11-09T05:40:23,162 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN; state=OFFLINE, location=e09398052c91,34331,1731130820076; forceNewPlan=false, retain=false 2024-11-09T05:40:23,163 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=706d9970ba371fdec8f2a079940ca3ae, ASSIGN 2024-11-09T05:40:23,165 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=706d9970ba371fdec8f2a079940ca3ae, ASSIGN; state=OFFLINE, location=e09398052c91,40411,1731130820128; forceNewPlan=false, retain=false 2024-11-09T05:40:23,166 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6ea3613715cd8a321124b73881db15ce, ASSIGN 2024-11-09T05:40:23,168 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6ea3613715cd8a321124b73881db15ce, ASSIGN; state=OFFLINE, location=e09398052c91,44035,1731130819959; forceNewPlan=false, retain=false 2024-11-09T05:40:23,169 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:23,170 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN; state=OFFLINE, location=e09398052c91,34331,1731130820076; forceNewPlan=false, retain=false 2024-11-09T05:40:23,171 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:23,172 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN; state=OFFLINE, location=e09398052c91,34331,1731130820076; forceNewPlan=false, retain=false 2024-11-09T05:40:23,174 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=df09fc3517c167e05c71f65345ca6b80, ASSIGN 2024-11-09T05:40:23,175 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=df09fc3517c167e05c71f65345ca6b80, ASSIGN; state=OFFLINE, location=e09398052c91,40411,1731130820128; forceNewPlan=false, retain=false 2024-11-09T05:40:23,176 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=22143fefe98480e8609aa64fb5bbe23e, ASSIGN 2024-11-09T05:40:23,177 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=22143fefe98480e8609aa64fb5bbe23e, ASSIGN; state=OFFLINE, location=e09398052c91,44035,1731130819959; forceNewPlan=false, retain=false 2024-11-09T05:40:23,178 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:23,180 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN; state=OFFLINE, location=e09398052c91,34331,1731130820076; forceNewPlan=false, retain=false 2024-11-09T05:40:23,181 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=51844aa86315c53347f864258b9b5926, ASSIGN 2024-11-09T05:40:23,182 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=51844aa86315c53347f864258b9b5926, ASSIGN; state=OFFLINE, location=e09398052c91,40411,1731130820128; forceNewPlan=false, retain=false 2024-11-09T05:40:23,183 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:23,185 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN; state=OFFLINE, location=e09398052c91,34331,1731130820076; forceNewPlan=false, retain=false 2024-11-09T05:40:23,186 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9fee264e99db7cad2e2fb211d7666c11, ASSIGN 2024-11-09T05:40:23,187 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9fee264e99db7cad2e2fb211d7666c11, ASSIGN; state=OFFLINE, location=e09398052c91,44035,1731130819959; forceNewPlan=false, retain=false 2024-11-09T05:40:23,188 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=056a4a6d72fcadc66bad84500d464f5b, ASSIGN 2024-11-09T05:40:23,189 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=056a4a6d72fcadc66bad84500d464f5b, ASSIGN; state=OFFLINE, location=e09398052c91,40411,1731130820128; forceNewPlan=false, retain=false 2024-11-09T05:40:23,279 INFO [e09398052c91:34809 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-09T05:40:23,280 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPENING, regionLocation=e09398052c91,44035,1731130819959 2024-11-09T05:40:23,281 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=0f1e5e5fd2628827f76a773887689bc8, regionState=OPENING, regionLocation=e09398052c91,44035,1731130819959 2024-11-09T05:40:23,282 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=c06857dd8e828d035329855bf73ed41f, regionState=OPENING, regionLocation=e09398052c91,44035,1731130819959 2024-11-09T05:40:23,282 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=d014bc791a8473500fbbfa044123f955, regionState=OPENING, regionLocation=e09398052c91,44035,1731130819959 2024-11-09T05:40:23,283 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=6c8538a5c08a601164d6d90f17409b33, regionState=OPENING, regionLocation=e09398052c91,44035,1731130819959 2024-11-09T05:40:23,283 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=6ea3613715cd8a321124b73881db15ce, regionState=OPENING, regionLocation=e09398052c91,44035,1731130819959 2024-11-09T05:40:23,284 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=22143fefe98480e8609aa64fb5bbe23e, regionState=OPENING, regionLocation=e09398052c91,44035,1731130819959 2024-11-09T05:40:23,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d923c8e1af5892d9575aa4b3336f28f9, ASSIGN because future has completed 2024-11-09T05:40:23,284 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=9fee264e99db7cad2e2fb211d7666c11, regionState=OPENING, regionLocation=e09398052c91,44035,1731130819959 2024-11-09T05:40:23,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure d923c8e1af5892d9575aa4b3336f28f9, server=e09398052c91,44035,1731130819959}] 2024-11-09T05:40:23,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0f1e5e5fd2628827f76a773887689bc8, ASSIGN because future has completed 2024-11-09T05:40:23,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c06857dd8e828d035329855bf73ed41f, ASSIGN because future has completed 2024-11-09T05:40:23,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d014bc791a8473500fbbfa044123f955, ASSIGN because future has completed 2024-11-09T05:40:23,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure c06857dd8e828d035329855bf73ed41f, server=e09398052c91,44035,1731130819959}] 2024-11-09T05:40:23,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6c8538a5c08a601164d6d90f17409b33, ASSIGN because future has completed 2024-11-09T05:40:23,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6ea3613715cd8a321124b73881db15ce, ASSIGN because future has completed 2024-11-09T05:40:23,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=22143fefe98480e8609aa64fb5bbe23e, ASSIGN because future has completed 2024-11-09T05:40:23,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22143fefe98480e8609aa64fb5bbe23e, server=e09398052c91,44035,1731130819959}] 2024-11-09T05:40:23,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9fee264e99db7cad2e2fb211d7666c11, ASSIGN because future has completed 2024-11-09T05:40:23,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fee264e99db7cad2e2fb211d7666c11, server=e09398052c91,44035,1731130819959}] 2024-11-09T05:40:23,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ea3613715cd8a321124b73881db15ce, server=e09398052c91,44035,1731130819959}] 2024-11-09T05:40:23,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c8538a5c08a601164d6d90f17409b33, server=e09398052c91,44035,1731130819959}] 2024-11-09T05:40:23,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure d014bc791a8473500fbbfa044123f955, server=e09398052c91,44035,1731130819959}] 2024-11-09T05:40:23,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0f1e5e5fd2628827f76a773887689bc8, server=e09398052c91,44035,1731130819959}] 2024-11-09T05:40:23,329 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=6874567d13af4b499b6eb873decadd10, regionState=OPENING, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:23,330 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPENING, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:23,331 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=2339918a4a54d7d8274a43da565fe2a2, regionState=OPENING, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:23,331 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPENING, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:23,332 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=ce006577ade7d81e0bf87827254771f5, regionState=OPENING, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:23,333 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=706d9970ba371fdec8f2a079940ca3ae, regionState=OPENING, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:23,333 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=df09fc3517c167e05c71f65345ca6b80, regionState=OPENING, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:23,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6874567d13af4b499b6eb873decadd10, ASSIGN because future has completed 2024-11-09T05:40:23,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6874567d13af4b499b6eb873decadd10, server=e09398052c91,40411,1731130820128}] 2024-11-09T05:40:23,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1cc6d417b0bdc1061e6457e90a27badd, ASSIGN because future has completed 2024-11-09T05:40:23,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2339918a4a54d7d8274a43da565fe2a2, ASSIGN because future has completed 2024-11-09T05:40:23,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8d8a7929b729c2e22434e4a1d52e461a, ASSIGN because future has completed 2024-11-09T05:40:23,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2339918a4a54d7d8274a43da565fe2a2, server=e09398052c91,40411,1731130820128}] 2024-11-09T05:40:23,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ce006577ade7d81e0bf87827254771f5, ASSIGN because future has completed 2024-11-09T05:40:23,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=706d9970ba371fdec8f2a079940ca3ae, ASSIGN because future has completed 2024-11-09T05:40:23,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=df09fc3517c167e05c71f65345ca6b80, ASSIGN because future has completed 2024-11-09T05:40:23,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure df09fc3517c167e05c71f65345ca6b80, server=e09398052c91,40411,1731130820128}] 2024-11-09T05:40:23,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 706d9970ba371fdec8f2a079940ca3ae, server=e09398052c91,40411,1731130820128}] 2024-11-09T05:40:23,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure ce006577ade7d81e0bf87827254771f5, server=e09398052c91,40411,1731130820128}] 2024-11-09T05:40:23,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d8a7929b729c2e22434e4a1d52e461a, server=e09398052c91,40411,1731130820128}] 2024-11-09T05:40:23,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cc6d417b0bdc1061e6457e90a27badd, server=e09398052c91,40411,1731130820128}] 2024-11-09T05:40:23,370 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=51844aa86315c53347f864258b9b5926, regionState=OPENING, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:23,371 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=056a4a6d72fcadc66bad84500d464f5b, regionState=OPENING, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:23,372 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, regionLocation=e09398052c91,34331,1731130820076 2024-11-09T05:40:23,373 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=532f29d232f4841336f315b0406fe4af, regionState=OPENING, regionLocation=e09398052c91,34331,1731130820076 2024-11-09T05:40:23,373 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, regionLocation=e09398052c91,34331,1731130820076 2024-11-09T05:40:23,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=51844aa86315c53347f864258b9b5926, ASSIGN because future has completed 2024-11-09T05:40:23,374 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPENING, regionLocation=e09398052c91,34331,1731130820076 2024-11-09T05:40:23,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 51844aa86315c53347f864258b9b5926, server=e09398052c91,40411,1731130820128}] 2024-11-09T05:40:23,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=056a4a6d72fcadc66bad84500d464f5b, ASSIGN because future has completed 2024-11-09T05:40:23,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN because future has completed 2024-11-09T05:40:23,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN because future has completed 2024-11-09T05:40:23,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN because future has completed 2024-11-09T05:40:23,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 532f29d232f4841336f315b0406fe4af, server=e09398052c91,34331,1731130820076}] 2024-11-09T05:40:23,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, ASSIGN because future has completed 2024-11-09T05:40:23,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d63f643dc4acf4d7c54bcb479b29df0, server=e09398052c91,34331,1731130820076}] 2024-11-09T05:40:23,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8714188f3856e307485ef7426df7ea3c, server=e09398052c91,34331,1731130820076}] 2024-11-09T05:40:23,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure b609b56af9f6fc87d9a153736ec0c8d1, server=e09398052c91,34331,1731130820076}] 2024-11-09T05:40:23,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 056a4a6d72fcadc66bad84500d464f5b, server=e09398052c91,40411,1731130820128}] 2024-11-09T05:40:23,404 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=12126c11ff675bc823b55beab9656286, regionState=OPENING, regionLocation=e09398052c91,34331,1731130820076 2024-11-09T05:40:23,405 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=348c820a04c95623395fb88f37fad187, regionState=OPENING, regionLocation=e09398052c91,34331,1731130820076 2024-11-09T05:40:23,405 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, regionLocation=e09398052c91,34331,1731130820076 2024-11-09T05:40:23,406 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, regionLocation=e09398052c91,34331,1731130820076 2024-11-09T05:40:23,407 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, regionLocation=e09398052c91,34331,1731130820076 2024-11-09T05:40:23,407 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN because future has completed 2024-11-09T05:40:23,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12126c11ff675bc823b55beab9656286, server=e09398052c91,34331,1731130820076}] 2024-11-09T05:40:23,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN because future has completed 2024-11-09T05:40:23,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN because future has completed 2024-11-09T05:40:23,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN because future has completed 2024-11-09T05:40:23,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN because future has completed 2024-11-09T05:40:23,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,34331,1731130820076}] 2024-11-09T05:40:23,422 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4485a6989fa449d3718b74dd5bef8f3, server=e09398052c91,34331,1731130820076}] 2024-11-09T05:40:23,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd305d310e01738dc5cbabbfddb8eb5c, server=e09398052c91,34331,1731130820076}] 2024-11-09T05:40:23,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 348c820a04c95623395fb88f37fad187, server=e09398052c91,34331,1731130820076}] 2024-11-09T05:40:23,439 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T05:40:23,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58433, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T05:40:23,447 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,ppp,1731130822466.d014bc791a8473500fbbfa044123f955. 2024-11-09T05:40:23,448 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => d014bc791a8473500fbbfa044123f955, NAME => 'test,ppp,1731130822466.d014bc791a8473500fbbfa044123f955.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-09T05:40:23,449 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d014bc791a8473500fbbfa044123f955 2024-11-09T05:40:23,449 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,ppp,1731130822466.d014bc791a8473500fbbfa044123f955.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,449 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for d014bc791a8473500fbbfa044123f955 2024-11-09T05:40:23,449 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for d014bc791a8473500fbbfa044123f955 2024-11-09T05:40:23,451 INFO [StoreOpener-d014bc791a8473500fbbfa044123f955-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d014bc791a8473500fbbfa044123f955 2024-11-09T05:40:23,453 INFO [StoreOpener-d014bc791a8473500fbbfa044123f955-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d014bc791a8473500fbbfa044123f955 columnFamilyName family 2024-11-09T05:40:23,454 DEBUG [StoreOpener-d014bc791a8473500fbbfa044123f955-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,454 INFO [StoreOpener-d014bc791a8473500fbbfa044123f955-1 {}] regionserver.HStore(327): Store=d014bc791a8473500fbbfa044123f955/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,455 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for d014bc791a8473500fbbfa044123f955 2024-11-09T05:40:23,456 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/d014bc791a8473500fbbfa044123f955 2024-11-09T05:40:23,457 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/d014bc791a8473500fbbfa044123f955 2024-11-09T05:40:23,457 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for d014bc791a8473500fbbfa044123f955 2024-11-09T05:40:23,457 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for d014bc791a8473500fbbfa044123f955 2024-11-09T05:40:23,460 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for d014bc791a8473500fbbfa044123f955 2024-11-09T05:40:23,463 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/d014bc791a8473500fbbfa044123f955/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,465 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened d014bc791a8473500fbbfa044123f955; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74170710, jitterRate=0.1052297055721283}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,465 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d014bc791a8473500fbbfa044123f955 2024-11-09T05:40:23,466 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for d014bc791a8473500fbbfa044123f955: Running coprocessor pre-open hook at 1731130823449Writing region info on filesystem at 1731130823449Initializing all the Stores at 1731130823451 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823451Cleaning up temporary data from old regions at 1731130823457 (+6 ms)Running coprocessor post-open hooks at 1731130823465 (+8 ms)Region opened successfully at 1731130823466 (+1 ms) 2024-11-09T05:40:23,468 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1731130822466.d014bc791a8473500fbbfa044123f955., pid=37, masterSystemTime=1731130823439 2024-11-09T05:40:23,471 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1731130822466.d014bc791a8473500fbbfa044123f955. 2024-11-09T05:40:23,471 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,ppp,1731130822466.d014bc791a8473500fbbfa044123f955. 2024-11-09T05:40:23,471 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,bbb,1731130822466.9fee264e99db7cad2e2fb211d7666c11. 2024-11-09T05:40:23,472 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 9fee264e99db7cad2e2fb211d7666c11, NAME => 'test,bbb,1731130822466.9fee264e99db7cad2e2fb211d7666c11.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-09T05:40:23,472 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9fee264e99db7cad2e2fb211d7666c11 2024-11-09T05:40:23,472 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=d014bc791a8473500fbbfa044123f955, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,44035,1731130819959 2024-11-09T05:40:23,472 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,bbb,1731130822466.9fee264e99db7cad2e2fb211d7666c11.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,472 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 9fee264e99db7cad2e2fb211d7666c11 2024-11-09T05:40:23,472 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 9fee264e99db7cad2e2fb211d7666c11 2024-11-09T05:40:23,475 INFO [StoreOpener-9fee264e99db7cad2e2fb211d7666c11-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9fee264e99db7cad2e2fb211d7666c11 2024-11-09T05:40:23,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure d014bc791a8473500fbbfa044123f955, server=e09398052c91,44035,1731130819959 because future has completed 2024-11-09T05:40:23,478 INFO [StoreOpener-9fee264e99db7cad2e2fb211d7666c11-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9fee264e99db7cad2e2fb211d7666c11 columnFamilyName family 2024-11-09T05:40:23,478 DEBUG [StoreOpener-9fee264e99db7cad2e2fb211d7666c11-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,478 INFO [StoreOpener-9fee264e99db7cad2e2fb211d7666c11-1 {}] regionserver.HStore(327): Store=9fee264e99db7cad2e2fb211d7666c11/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,479 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 9fee264e99db7cad2e2fb211d7666c11 2024-11-09T05:40:23,480 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/9fee264e99db7cad2e2fb211d7666c11 2024-11-09T05:40:23,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=20 2024-11-09T05:40:23,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure d014bc791a8473500fbbfa044123f955, server=e09398052c91,44035,1731130819959 in 157 msec 2024-11-09T05:40:23,481 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/9fee264e99db7cad2e2fb211d7666c11 2024-11-09T05:40:23,481 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 9fee264e99db7cad2e2fb211d7666c11 2024-11-09T05:40:23,481 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 9fee264e99db7cad2e2fb211d7666c11 2024-11-09T05:40:23,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d014bc791a8473500fbbfa044123f955, ASSIGN in 364 msec 2024-11-09T05:40:23,484 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 9fee264e99db7cad2e2fb211d7666c11 2024-11-09T05:40:23,487 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/9fee264e99db7cad2e2fb211d7666c11/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,488 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T05:40:23,488 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 9fee264e99db7cad2e2fb211d7666c11; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59330378, jitterRate=-0.11590847373008728}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,488 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9fee264e99db7cad2e2fb211d7666c11 2024-11-09T05:40:23,488 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 9fee264e99db7cad2e2fb211d7666c11: Running coprocessor pre-open hook at 1731130823473Writing region info on filesystem at 1731130823473Initializing all the Stores at 1731130823474 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823474Cleaning up temporary data from old regions at 1731130823481 (+7 ms)Running coprocessor post-open hooks at 1731130823488 (+7 ms)Region opened successfully at 1731130823488 2024-11-09T05:40:23,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46381, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T05:40:23,491 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731130822466.9fee264e99db7cad2e2fb211d7666c11., pid=34, masterSystemTime=1731130823439 2024-11-09T05:40:23,493 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731130822466.9fee264e99db7cad2e2fb211d7666c11. 2024-11-09T05:40:23,493 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,bbb,1731130822466.9fee264e99db7cad2e2fb211d7666c11. 2024-11-09T05:40:23,494 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,xxx,1731130822466.d923c8e1af5892d9575aa4b3336f28f9. 2024-11-09T05:40:23,494 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => d923c8e1af5892d9575aa4b3336f28f9, NAME => 'test,xxx,1731130822466.d923c8e1af5892d9575aa4b3336f28f9.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-09T05:40:23,494 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d923c8e1af5892d9575aa4b3336f28f9 2024-11-09T05:40:23,494 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,xxx,1731130822466.d923c8e1af5892d9575aa4b3336f28f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,495 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for d923c8e1af5892d9575aa4b3336f28f9 2024-11-09T05:40:23,495 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for d923c8e1af5892d9575aa4b3336f28f9 2024-11-09T05:40:23,495 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,44035,1731130819959 2024-11-09T05:40:23,497 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,zzz,1731130822466.6874567d13af4b499b6eb873decadd10. 2024-11-09T05:40:23,497 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 6874567d13af4b499b6eb873decadd10, NAME => 'test,zzz,1731130822466.6874567d13af4b499b6eb873decadd10.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-09T05:40:23,498 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6874567d13af4b499b6eb873decadd10 2024-11-09T05:40:23,498 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,zzz,1731130822466.6874567d13af4b499b6eb873decadd10.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,498 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 6874567d13af4b499b6eb873decadd10 2024-11-09T05:40:23,498 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 6874567d13af4b499b6eb873decadd10 2024-11-09T05:40:23,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fee264e99db7cad2e2fb211d7666c11, server=e09398052c91,44035,1731130819959 because future has completed 2024-11-09T05:40:23,500 INFO [StoreOpener-d923c8e1af5892d9575aa4b3336f28f9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d923c8e1af5892d9575aa4b3336f28f9 2024-11-09T05:40:23,501 INFO [StoreOpener-6874567d13af4b499b6eb873decadd10-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6874567d13af4b499b6eb873decadd10 2024-11-09T05:40:23,502 INFO [StoreOpener-d923c8e1af5892d9575aa4b3336f28f9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d923c8e1af5892d9575aa4b3336f28f9 columnFamilyName family 2024-11-09T05:40:23,502 DEBUG [StoreOpener-d923c8e1af5892d9575aa4b3336f28f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=6 2024-11-09T05:40:23,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 9fee264e99db7cad2e2fb211d7666c11, server=e09398052c91,44035,1731130819959 in 194 msec 2024-11-09T05:40:23,504 INFO [StoreOpener-d923c8e1af5892d9575aa4b3336f28f9-1 {}] regionserver.HStore(327): Store=d923c8e1af5892d9575aa4b3336f28f9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,504 INFO [StoreOpener-6874567d13af4b499b6eb873decadd10-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6874567d13af4b499b6eb873decadd10 columnFamilyName family 2024-11-09T05:40:23,504 DEBUG [StoreOpener-6874567d13af4b499b6eb873decadd10-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,505 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for d923c8e1af5892d9575aa4b3336f28f9 2024-11-09T05:40:23,505 INFO [StoreOpener-6874567d13af4b499b6eb873decadd10-1 {}] regionserver.HStore(327): Store=6874567d13af4b499b6eb873decadd10/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,505 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 6874567d13af4b499b6eb873decadd10 2024-11-09T05:40:23,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9fee264e99db7cad2e2fb211d7666c11, ASSIGN in 388 msec 2024-11-09T05:40:23,506 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/d923c8e1af5892d9575aa4b3336f28f9 2024-11-09T05:40:23,506 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/d923c8e1af5892d9575aa4b3336f28f9 2024-11-09T05:40:23,507 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/6874567d13af4b499b6eb873decadd10 2024-11-09T05:40:23,507 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for d923c8e1af5892d9575aa4b3336f28f9 2024-11-09T05:40:23,507 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for d923c8e1af5892d9575aa4b3336f28f9 2024-11-09T05:40:23,507 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/6874567d13af4b499b6eb873decadd10 2024-11-09T05:40:23,508 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 6874567d13af4b499b6eb873decadd10 2024-11-09T05:40:23,508 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 6874567d13af4b499b6eb873decadd10 2024-11-09T05:40:23,509 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for d923c8e1af5892d9575aa4b3336f28f9 2024-11-09T05:40:23,510 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 6874567d13af4b499b6eb873decadd10 2024-11-09T05:40:23,513 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/d923c8e1af5892d9575aa4b3336f28f9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,514 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/6874567d13af4b499b6eb873decadd10/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,514 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened d923c8e1af5892d9575aa4b3336f28f9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60244980, jitterRate=-0.10227984189987183}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,514 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d923c8e1af5892d9575aa4b3336f28f9 2024-11-09T05:40:23,514 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for d923c8e1af5892d9575aa4b3336f28f9: Running coprocessor pre-open hook at 1731130823495Writing region info on filesystem at 1731130823495Initializing all the Stores at 1731130823496 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823496Cleaning up temporary data from old regions at 1731130823507 (+11 ms)Running coprocessor post-open hooks at 1731130823514 (+7 ms)Region opened successfully at 1731130823514 2024-11-09T05:40:23,515 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 6874567d13af4b499b6eb873decadd10; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63573570, jitterRate=-0.052679985761642456}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,516 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6874567d13af4b499b6eb873decadd10 2024-11-09T05:40:23,516 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 6874567d13af4b499b6eb873decadd10: Running coprocessor pre-open hook at 1731130823498Writing region info on filesystem at 1731130823498Initializing all the Stores at 1731130823499 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823499Cleaning up temporary data from old regions at 1731130823508 (+9 ms)Running coprocessor post-open hooks at 1731130823516 (+8 ms)Region opened successfully at 1731130823516 2024-11-09T05:40:23,516 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1731130822466.d923c8e1af5892d9575aa4b3336f28f9., pid=31, masterSystemTime=1731130823439 2024-11-09T05:40:23,517 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731130822466.6874567d13af4b499b6eb873decadd10., pid=39, masterSystemTime=1731130823488 2024-11-09T05:40:23,520 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1731130822466.d923c8e1af5892d9575aa4b3336f28f9. 2024-11-09T05:40:23,520 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,xxx,1731130822466.d923c8e1af5892d9575aa4b3336f28f9. 2024-11-09T05:40:23,520 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,jjj,1731130822466.6ea3613715cd8a321124b73881db15ce. 2024-11-09T05:40:23,520 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ea3613715cd8a321124b73881db15ce, NAME => 'test,jjj,1731130822466.6ea3613715cd8a321124b73881db15ce.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-09T05:40:23,521 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6ea3613715cd8a321124b73881db15ce 2024-11-09T05:40:23,521 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,jjj,1731130822466.6ea3613715cd8a321124b73881db15ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,521 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 6ea3613715cd8a321124b73881db15ce 2024-11-09T05:40:23,521 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 6ea3613715cd8a321124b73881db15ce 2024-11-09T05:40:23,522 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,44035,1731130819959 2024-11-09T05:40:23,522 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731130822466.6874567d13af4b499b6eb873decadd10. 2024-11-09T05:40:23,522 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,zzz,1731130822466.6874567d13af4b499b6eb873decadd10. 2024-11-09T05:40:23,522 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,kkk,1731130822466.706d9970ba371fdec8f2a079940ca3ae. 2024-11-09T05:40:23,523 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 706d9970ba371fdec8f2a079940ca3ae, NAME => 'test,kkk,1731130822466.706d9970ba371fdec8f2a079940ca3ae.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-09T05:40:23,524 INFO [StoreOpener-6ea3613715cd8a321124b73881db15ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6ea3613715cd8a321124b73881db15ce 2024-11-09T05:40:23,524 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 706d9970ba371fdec8f2a079940ca3ae 2024-11-09T05:40:23,524 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,kkk,1731130822466.706d9970ba371fdec8f2a079940ca3ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,525 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 706d9970ba371fdec8f2a079940ca3ae 2024-11-09T05:40:23,525 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 706d9970ba371fdec8f2a079940ca3ae 2024-11-09T05:40:23,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure d923c8e1af5892d9575aa4b3336f28f9, server=e09398052c91,44035,1731130819959 because future has completed 2024-11-09T05:40:23,526 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=6874567d13af4b499b6eb873decadd10, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:23,527 INFO [StoreOpener-6ea3613715cd8a321124b73881db15ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ea3613715cd8a321124b73881db15ce columnFamilyName family 2024-11-09T05:40:23,527 DEBUG [StoreOpener-6ea3613715cd8a321124b73881db15ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,528 INFO [StoreOpener-6ea3613715cd8a321124b73881db15ce-1 {}] regionserver.HStore(327): Store=6ea3613715cd8a321124b73881db15ce/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,528 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 6ea3613715cd8a321124b73881db15ce 2024-11-09T05:40:23,528 INFO [StoreOpener-706d9970ba371fdec8f2a079940ca3ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 706d9970ba371fdec8f2a079940ca3ae 2024-11-09T05:40:23,529 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/6ea3613715cd8a321124b73881db15ce 2024-11-09T05:40:23,530 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/6ea3613715cd8a321124b73881db15ce 2024-11-09T05:40:23,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6874567d13af4b499b6eb873decadd10, server=e09398052c91,40411,1731130820128 because future has completed 2024-11-09T05:40:23,530 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 6ea3613715cd8a321124b73881db15ce 2024-11-09T05:40:23,530 INFO [StoreOpener-706d9970ba371fdec8f2a079940ca3ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 706d9970ba371fdec8f2a079940ca3ae columnFamilyName family 2024-11-09T05:40:23,530 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 6ea3613715cd8a321124b73881db15ce 2024-11-09T05:40:23,531 DEBUG [StoreOpener-706d9970ba371fdec8f2a079940ca3ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,531 INFO [StoreOpener-706d9970ba371fdec8f2a079940ca3ae-1 {}] regionserver.HStore(327): Store=706d9970ba371fdec8f2a079940ca3ae/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,531 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 706d9970ba371fdec8f2a079940ca3ae 2024-11-09T05:40:23,533 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/706d9970ba371fdec8f2a079940ca3ae 2024-11-09T05:40:23,533 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/706d9970ba371fdec8f2a079940ca3ae 2024-11-09T05:40:23,534 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 6ea3613715cd8a321124b73881db15ce 2024-11-09T05:40:23,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=28 2024-11-09T05:40:23,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure d923c8e1af5892d9575aa4b3336f28f9, server=e09398052c91,44035,1731130819959 in 245 msec 2024-11-09T05:40:23,535 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 706d9970ba371fdec8f2a079940ca3ae 2024-11-09T05:40:23,535 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 706d9970ba371fdec8f2a079940ca3ae 2024-11-09T05:40:23,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d923c8e1af5892d9575aa4b3336f28f9, ASSIGN in 419 msec 2024-11-09T05:40:23,539 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/6ea3613715cd8a321124b73881db15ce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,539 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 706d9970ba371fdec8f2a079940ca3ae 2024-11-09T05:40:23,540 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af. 2024-11-09T05:40:23,540 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 532f29d232f4841336f315b0406fe4af, NAME => 'test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-09T05:40:23,541 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 6ea3613715cd8a321124b73881db15ce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61463940, jitterRate=-0.08411592245101929}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,541 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6ea3613715cd8a321124b73881db15ce 2024-11-09T05:40:23,541 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 6ea3613715cd8a321124b73881db15ce: Running coprocessor pre-open hook at 1731130823522Writing region info on filesystem at 1731130823522Initializing all the Stores at 1731130823524 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823524Cleaning up temporary data from old regions at 1731130823531 (+7 ms)Running coprocessor post-open hooks at 1731130823541 (+10 ms)Region opened successfully at 1731130823541 2024-11-09T05:40:23,541 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:23,542 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,542 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:23,542 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:23,543 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731130822466.6ea3613715cd8a321124b73881db15ce., pid=35, masterSystemTime=1731130823439 2024-11-09T05:40:23,543 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/706d9970ba371fdec8f2a079940ca3ae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,544 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 706d9970ba371fdec8f2a079940ca3ae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74629183, jitterRate=0.11206148564815521}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,545 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 706d9970ba371fdec8f2a079940ca3ae 2024-11-09T05:40:23,545 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 706d9970ba371fdec8f2a079940ca3ae: Running coprocessor pre-open hook at 1731130823525Writing region info on filesystem at 1731130823525Initializing all the Stores at 1731130823527 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823527Cleaning up temporary data from old regions at 1731130823535 (+8 ms)Running coprocessor post-open hooks at 1731130823545 (+10 ms)Region opened successfully at 1731130823545 2024-11-09T05:40:23,546 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731130822466.706d9970ba371fdec8f2a079940ca3ae., pid=42, masterSystemTime=1731130823488 2024-11-09T05:40:23,546 INFO [StoreOpener-532f29d232f4841336f315b0406fe4af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:23,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=30 2024-11-09T05:40:23,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 6874567d13af4b499b6eb873decadd10, server=e09398052c91,40411,1731130820128 in 207 msec 2024-11-09T05:40:23,546 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731130822466.6ea3613715cd8a321124b73881db15ce. 2024-11-09T05:40:23,547 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,jjj,1731130822466.6ea3613715cd8a321124b73881db15ce. 2024-11-09T05:40:23,547 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,sss,1731130822466.c06857dd8e828d035329855bf73ed41f. 2024-11-09T05:40:23,547 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => c06857dd8e828d035329855bf73ed41f, NAME => 'test,sss,1731130822466.c06857dd8e828d035329855bf73ed41f.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-09T05:40:23,547 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c06857dd8e828d035329855bf73ed41f 2024-11-09T05:40:23,547 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,sss,1731130822466.c06857dd8e828d035329855bf73ed41f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,548 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for c06857dd8e828d035329855bf73ed41f 2024-11-09T05:40:23,548 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for c06857dd8e828d035329855bf73ed41f 2024-11-09T05:40:23,549 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1731130822466.706d9970ba371fdec8f2a079940ca3ae. 2024-11-09T05:40:23,549 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,kkk,1731130822466.706d9970ba371fdec8f2a079940ca3ae. 2024-11-09T05:40:23,549 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,ggg,1731130822466.df09fc3517c167e05c71f65345ca6b80. 2024-11-09T05:40:23,549 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => df09fc3517c167e05c71f65345ca6b80, NAME => 'test,ggg,1731130822466.df09fc3517c167e05c71f65345ca6b80.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-09T05:40:23,549 INFO [StoreOpener-532f29d232f4841336f315b0406fe4af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 532f29d232f4841336f315b0406fe4af columnFamilyName family 2024-11-09T05:40:23,550 DEBUG [StoreOpener-532f29d232f4841336f315b0406fe4af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,550 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test df09fc3517c167e05c71f65345ca6b80 2024-11-09T05:40:23,550 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,ggg,1731130822466.df09fc3517c167e05c71f65345ca6b80.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,550 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for df09fc3517c167e05c71f65345ca6b80 2024-11-09T05:40:23,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6874567d13af4b499b6eb873decadd10, ASSIGN in 431 msec 2024-11-09T05:40:23,550 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for df09fc3517c167e05c71f65345ca6b80 2024-11-09T05:40:23,550 INFO [StoreOpener-532f29d232f4841336f315b0406fe4af-1 {}] regionserver.HStore(327): Store=532f29d232f4841336f315b0406fe4af/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,551 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:23,551 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:23,553 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:23,553 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:23,554 INFO [StoreOpener-c06857dd8e828d035329855bf73ed41f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c06857dd8e828d035329855bf73ed41f 2024-11-09T05:40:23,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 706d9970ba371fdec8f2a079940ca3ae, server=e09398052c91,40411,1731130820128 because future has completed 2024-11-09T05:40:23,554 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:23,554 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:23,556 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,44035,1731130819959 2024-11-09T05:40:23,556 INFO [StoreOpener-c06857dd8e828d035329855bf73ed41f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c06857dd8e828d035329855bf73ed41f columnFamilyName family 2024-11-09T05:40:23,556 INFO [StoreOpener-df09fc3517c167e05c71f65345ca6b80-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region df09fc3517c167e05c71f65345ca6b80 2024-11-09T05:40:23,556 DEBUG [StoreOpener-c06857dd8e828d035329855bf73ed41f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,557 INFO [StoreOpener-c06857dd8e828d035329855bf73ed41f-1 {}] regionserver.HStore(327): Store=c06857dd8e828d035329855bf73ed41f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,558 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for c06857dd8e828d035329855bf73ed41f 2024-11-09T05:40:23,558 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:23,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ea3613715cd8a321124b73881db15ce, server=e09398052c91,44035,1731130819959 because future has completed 2024-11-09T05:40:23,559 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/c06857dd8e828d035329855bf73ed41f 2024-11-09T05:40:23,559 INFO [StoreOpener-df09fc3517c167e05c71f65345ca6b80-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df09fc3517c167e05c71f65345ca6b80 columnFamilyName family 2024-11-09T05:40:23,559 DEBUG [StoreOpener-df09fc3517c167e05c71f65345ca6b80-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,560 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/c06857dd8e828d035329855bf73ed41f 2024-11-09T05:40:23,560 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for c06857dd8e828d035329855bf73ed41f 2024-11-09T05:40:23,560 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for c06857dd8e828d035329855bf73ed41f 2024-11-09T05:40:23,561 INFO [StoreOpener-df09fc3517c167e05c71f65345ca6b80-1 {}] regionserver.HStore(327): Store=df09fc3517c167e05c71f65345ca6b80/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,561 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for df09fc3517c167e05c71f65345ca6b80 2024-11-09T05:40:23,561 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/532f29d232f4841336f315b0406fe4af/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,562 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/df09fc3517c167e05c71f65345ca6b80 2024-11-09T05:40:23,562 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 532f29d232f4841336f315b0406fe4af; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74626051, jitterRate=0.11201481521129608}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,563 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:23,563 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 532f29d232f4841336f315b0406fe4af: Running coprocessor pre-open hook at 1731130823542Writing region info on filesystem at 1731130823542Initializing all the Stores at 1731130823544 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823544Cleaning up temporary data from old regions at 1731130823554 (+10 ms)Running coprocessor post-open hooks at 1731130823563 (+9 ms)Region opened successfully at 1731130823563 2024-11-09T05:40:23,563 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/df09fc3517c167e05c71f65345ca6b80 2024-11-09T05:40:23,564 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for c06857dd8e828d035329855bf73ed41f 2024-11-09T05:40:23,564 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for df09fc3517c167e05c71f65345ca6b80 2024-11-09T05:40:23,564 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for df09fc3517c167e05c71f65345ca6b80 2024-11-09T05:40:23,564 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af., pid=47, masterSystemTime=1731130823533 2024-11-09T05:40:23,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=15 2024-11-09T05:40:23,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 706d9970ba371fdec8f2a079940ca3ae, server=e09398052c91,40411,1731130820128 in 209 msec 2024-11-09T05:40:23,568 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/c06857dd8e828d035329855bf73ed41f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,568 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af. 2024-11-09T05:40:23,568 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=706d9970ba371fdec8f2a079940ca3ae, ASSIGN in 449 msec 2024-11-09T05:40:23,568 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for df09fc3517c167e05c71f65345ca6b80 2024-11-09T05:40:23,569 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af. 2024-11-09T05:40:23,569 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,lll,1731130822466.12126c11ff675bc823b55beab9656286. 2024-11-09T05:40:23,569 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened c06857dd8e828d035329855bf73ed41f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68544031, jitterRate=0.02138565480709076}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,569 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 12126c11ff675bc823b55beab9656286, NAME => 'test,lll,1731130822466.12126c11ff675bc823b55beab9656286.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-09T05:40:23,569 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c06857dd8e828d035329855bf73ed41f 2024-11-09T05:40:23,569 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for c06857dd8e828d035329855bf73ed41f: Running coprocessor pre-open hook at 1731130823548Writing region info on filesystem at 1731130823548Initializing all the Stores at 1731130823551 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823551Cleaning up temporary data from old regions at 1731130823560 (+9 ms)Running coprocessor post-open hooks at 1731130823569 (+9 ms)Region opened successfully at 1731130823569 2024-11-09T05:40:23,570 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 12126c11ff675bc823b55beab9656286 2024-11-09T05:40:23,570 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,lll,1731130822466.12126c11ff675bc823b55beab9656286.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,570 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 12126c11ff675bc823b55beab9656286 2024-11-09T05:40:23,570 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 12126c11ff675bc823b55beab9656286 2024-11-09T05:40:23,573 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/df09fc3517c167e05c71f65345ca6b80/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,573 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened df09fc3517c167e05c71f65345ca6b80; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67241888, jitterRate=0.0019822120666503906}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,573 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for df09fc3517c167e05c71f65345ca6b80 2024-11-09T05:40:23,573 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for df09fc3517c167e05c71f65345ca6b80: Running coprocessor pre-open hook at 1731130823550Writing region info on filesystem at 1731130823550Initializing all the Stores at 1731130823551 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823551Cleaning up temporary data from old regions at 1731130823564 (+13 ms)Running coprocessor post-open hooks at 1731130823573 (+9 ms)Region opened successfully at 1731130823573 2024-11-09T05:40:23,574 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1731130822466.df09fc3517c167e05c71f65345ca6b80., pid=41, masterSystemTime=1731130823488 2024-11-09T05:40:23,574 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731130822466.c06857dd8e828d035329855bf73ed41f., pid=32, masterSystemTime=1731130823439 2024-11-09T05:40:23,574 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=532f29d232f4841336f315b0406fe4af, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,34331,1731130820076 2024-11-09T05:40:23,576 INFO [StoreOpener-12126c11ff675bc823b55beab9656286-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 12126c11ff675bc823b55beab9656286 2024-11-09T05:40:23,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 532f29d232f4841336f315b0406fe4af, server=e09398052c91,34331,1731130820076 because future has completed 2024-11-09T05:40:23,578 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1731130822466.df09fc3517c167e05c71f65345ca6b80. 2024-11-09T05:40:23,578 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,ggg,1731130822466.df09fc3517c167e05c71f65345ca6b80. 2024-11-09T05:40:23,578 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,,1731130822466.056a4a6d72fcadc66bad84500d464f5b. 2024-11-09T05:40:23,579 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 056a4a6d72fcadc66bad84500d464f5b, NAME => 'test,,1731130822466.056a4a6d72fcadc66bad84500d464f5b.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-09T05:40:23,579 INFO [StoreOpener-12126c11ff675bc823b55beab9656286-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12126c11ff675bc823b55beab9656286 columnFamilyName family 2024-11-09T05:40:23,579 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 056a4a6d72fcadc66bad84500d464f5b 2024-11-09T05:40:23,579 DEBUG [StoreOpener-12126c11ff675bc823b55beab9656286-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,579 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,,1731130822466.056a4a6d72fcadc66bad84500d464f5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,579 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 056a4a6d72fcadc66bad84500d464f5b 2024-11-09T05:40:23,579 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 056a4a6d72fcadc66bad84500d464f5b 2024-11-09T05:40:23,580 INFO [StoreOpener-12126c11ff675bc823b55beab9656286-1 {}] regionserver.HStore(327): Store=12126c11ff675bc823b55beab9656286/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,580 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 12126c11ff675bc823b55beab9656286 2024-11-09T05:40:23,580 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731130822466.c06857dd8e828d035329855bf73ed41f. 2024-11-09T05:40:23,580 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,sss,1731130822466.c06857dd8e828d035329855bf73ed41f. 2024-11-09T05:40:23,580 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,vvv,1731130822466.0f1e5e5fd2628827f76a773887689bc8. 2024-11-09T05:40:23,581 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 0f1e5e5fd2628827f76a773887689bc8, NAME => 'test,vvv,1731130822466.0f1e5e5fd2628827f76a773887689bc8.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-09T05:40:23,581 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0f1e5e5fd2628827f76a773887689bc8 2024-11-09T05:40:23,581 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,vvv,1731130822466.0f1e5e5fd2628827f76a773887689bc8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,581 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 0f1e5e5fd2628827f76a773887689bc8 2024-11-09T05:40:23,581 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 0f1e5e5fd2628827f76a773887689bc8 2024-11-09T05:40:23,582 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/12126c11ff675bc823b55beab9656286 2024-11-09T05:40:23,583 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/12126c11ff675bc823b55beab9656286 2024-11-09T05:40:23,583 INFO [StoreOpener-056a4a6d72fcadc66bad84500d464f5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 056a4a6d72fcadc66bad84500d464f5b 2024-11-09T05:40:23,583 INFO [StoreOpener-0f1e5e5fd2628827f76a773887689bc8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0f1e5e5fd2628827f76a773887689bc8 2024-11-09T05:40:23,583 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 12126c11ff675bc823b55beab9656286 2024-11-09T05:40:23,583 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 12126c11ff675bc823b55beab9656286 2024-11-09T05:40:23,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=14 2024-11-09T05:40:23,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 6ea3613715cd8a321124b73881db15ce, server=e09398052c91,44035,1731130819959 in 268 msec 2024-11-09T05:40:23,585 INFO [StoreOpener-056a4a6d72fcadc66bad84500d464f5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 056a4a6d72fcadc66bad84500d464f5b columnFamilyName family 2024-11-09T05:40:23,585 DEBUG [StoreOpener-056a4a6d72fcadc66bad84500d464f5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,586 INFO [StoreOpener-056a4a6d72fcadc66bad84500d464f5b-1 {}] regionserver.HStore(327): Store=056a4a6d72fcadc66bad84500d464f5b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,586 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 12126c11ff675bc823b55beab9656286 2024-11-09T05:40:23,586 INFO [StoreOpener-0f1e5e5fd2628827f76a773887689bc8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f1e5e5fd2628827f76a773887689bc8 columnFamilyName family 2024-11-09T05:40:23,586 DEBUG [StoreOpener-0f1e5e5fd2628827f76a773887689bc8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,587 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 056a4a6d72fcadc66bad84500d464f5b 2024-11-09T05:40:23,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6ea3613715cd8a321124b73881db15ce, ASSIGN in 468 msec 2024-11-09T05:40:23,587 INFO [StoreOpener-0f1e5e5fd2628827f76a773887689bc8-1 {}] regionserver.HStore(327): Store=0f1e5e5fd2628827f76a773887689bc8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,587 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 0f1e5e5fd2628827f76a773887689bc8 2024-11-09T05:40:23,588 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,44035,1731130819959 2024-11-09T05:40:23,590 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/056a4a6d72fcadc66bad84500d464f5b 2024-11-09T05:40:23,590 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/0f1e5e5fd2628827f76a773887689bc8 2024-11-09T05:40:23,591 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/056a4a6d72fcadc66bad84500d464f5b 2024-11-09T05:40:23,591 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/0f1e5e5fd2628827f76a773887689bc8 2024-11-09T05:40:23,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure c06857dd8e828d035329855bf73ed41f, server=e09398052c91,44035,1731130819959 because future has completed 2024-11-09T05:40:23,592 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 056a4a6d72fcadc66bad84500d464f5b 2024-11-09T05:40:23,592 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 056a4a6d72fcadc66bad84500d464f5b 2024-11-09T05:40:23,592 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 0f1e5e5fd2628827f76a773887689bc8 2024-11-09T05:40:23,592 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/12126c11ff675bc823b55beab9656286/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,592 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 0f1e5e5fd2628827f76a773887689bc8 2024-11-09T05:40:23,593 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 12126c11ff675bc823b55beab9656286; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63175417, jitterRate=-0.05861292779445648}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,593 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 12126c11ff675bc823b55beab9656286 2024-11-09T05:40:23,593 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 12126c11ff675bc823b55beab9656286: Running coprocessor pre-open hook at 1731130823571Writing region info on filesystem at 1731130823571Initializing all the Stores at 1731130823572 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823572Cleaning up temporary data from old regions at 1731130823583 (+11 ms)Running coprocessor post-open hooks at 1731130823593 (+10 ms)Region opened successfully at 1731130823593 2024-11-09T05:40:23,594 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731130822466.12126c11ff675bc823b55beab9656286., pid=52, masterSystemTime=1731130823533 2024-11-09T05:40:23,595 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 0f1e5e5fd2628827f76a773887689bc8 2024-11-09T05:40:23,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=25 2024-11-09T05:40:23,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 532f29d232f4841336f315b0406fe4af, server=e09398052c91,34331,1731130820076 in 213 msec 2024-11-09T05:40:23,598 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731130822466.12126c11ff675bc823b55beab9656286. 2024-11-09T05:40:23,598 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,lll,1731130822466.12126c11ff675bc823b55beab9656286. 2024-11-09T05:40:23,598 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c. 2024-11-09T05:40:23,598 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => bd305d310e01738dc5cbabbfddb8eb5c, NAME => 'test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-09T05:40:23,599 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:23,599 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 056a4a6d72fcadc66bad84500d464f5b 2024-11-09T05:40:23,599 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,599 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:23,599 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:23,599 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=12126c11ff675bc823b55beab9656286, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,34331,1731130820076 2024-11-09T05:40:23,600 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/0f1e5e5fd2628827f76a773887689bc8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,601 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 0f1e5e5fd2628827f76a773887689bc8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60904743, jitterRate=-0.09244860708713531}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,602 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0f1e5e5fd2628827f76a773887689bc8 2024-11-09T05:40:23,602 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 0f1e5e5fd2628827f76a773887689bc8: Running coprocessor pre-open hook at 1731130823581Writing region info on filesystem at 1731130823581Initializing all the Stores at 1731130823583 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823583Cleaning up temporary data from old regions at 1731130823592 (+9 ms)Running coprocessor post-open hooks at 1731130823602 (+10 ms)Region opened successfully at 1731130823602 2024-11-09T05:40:23,602 INFO [StoreOpener-bd305d310e01738dc5cbabbfddb8eb5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:23,603 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731130822466.0f1e5e5fd2628827f76a773887689bc8., pid=38, masterSystemTime=1731130823439 2024-11-09T05:40:23,603 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12126c11ff675bc823b55beab9656286, server=e09398052c91,34331,1731130820076 because future has completed 2024-11-09T05:40:23,604 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/056a4a6d72fcadc66bad84500d464f5b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,605 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 056a4a6d72fcadc66bad84500d464f5b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59116911, jitterRate=-0.11908937990665436}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,605 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 056a4a6d72fcadc66bad84500d464f5b 2024-11-09T05:40:23,605 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 056a4a6d72fcadc66bad84500d464f5b: Running coprocessor pre-open hook at 1731130823579Writing region info on filesystem at 1731130823579Initializing all the Stores at 1731130823582 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823582Cleaning up temporary data from old regions at 1731130823592 (+10 ms)Running coprocessor post-open hooks at 1731130823605 (+13 ms)Region opened successfully at 1731130823605 2024-11-09T05:40:23,606 INFO [StoreOpener-bd305d310e01738dc5cbabbfddb8eb5c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd305d310e01738dc5cbabbfddb8eb5c columnFamilyName family 2024-11-09T05:40:23,606 DEBUG [StoreOpener-bd305d310e01738dc5cbabbfddb8eb5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,606 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731130822466.0f1e5e5fd2628827f76a773887689bc8. 2024-11-09T05:40:23,607 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,vvv,1731130822466.0f1e5e5fd2628827f76a773887689bc8. 2024-11-09T05:40:23,607 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731130822466.056a4a6d72fcadc66bad84500d464f5b., pid=51, masterSystemTime=1731130823488 2024-11-09T05:40:23,607 INFO [StoreOpener-bd305d310e01738dc5cbabbfddb8eb5c-1 {}] regionserver.HStore(327): Store=bd305d310e01738dc5cbabbfddb8eb5c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,607 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,mmm,1731130822466.6c8538a5c08a601164d6d90f17409b33. 2024-11-09T05:40:23,607 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:23,607 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 6c8538a5c08a601164d6d90f17409b33, NAME => 'test,mmm,1731130822466.6c8538a5c08a601164d6d90f17409b33.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-09T05:40:23,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN in 487 msec 2024-11-09T05:40:23,607 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6c8538a5c08a601164d6d90f17409b33 2024-11-09T05:40:23,608 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,mmm,1731130822466.6c8538a5c08a601164d6d90f17409b33.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,608 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 6c8538a5c08a601164d6d90f17409b33 2024-11-09T05:40:23,608 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 6c8538a5c08a601164d6d90f17409b33 2024-11-09T05:40:23,608 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:23,609 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:23,610 INFO [StoreOpener-6c8538a5c08a601164d6d90f17409b33-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6c8538a5c08a601164d6d90f17409b33 2024-11-09T05:40:23,611 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731130822466.056a4a6d72fcadc66bad84500d464f5b. 2024-11-09T05:40:23,611 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,,1731130822466.056a4a6d72fcadc66bad84500d464f5b. 2024-11-09T05:40:23,611 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:23,610 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,44035,1731130819959 2024-11-09T05:40:23,611 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,ddd,1731130822466.51844aa86315c53347f864258b9b5926. 2024-11-09T05:40:23,611 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:23,611 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 51844aa86315c53347f864258b9b5926, NAME => 'test,ddd,1731130822466.51844aa86315c53347f864258b9b5926.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-09T05:40:23,612 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 51844aa86315c53347f864258b9b5926 2024-11-09T05:40:23,612 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,ddd,1731130822466.51844aa86315c53347f864258b9b5926.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,612 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 51844aa86315c53347f864258b9b5926 2024-11-09T05:40:23,612 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 51844aa86315c53347f864258b9b5926 2024-11-09T05:40:23,614 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0f1e5e5fd2628827f76a773887689bc8, server=e09398052c91,44035,1731130819959 because future has completed 2024-11-09T05:40:23,615 INFO [StoreOpener-6c8538a5c08a601164d6d90f17409b33-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c8538a5c08a601164d6d90f17409b33 columnFamilyName family 2024-11-09T05:40:23,615 DEBUG [StoreOpener-6c8538a5c08a601164d6d90f17409b33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,616 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:23,617 INFO [StoreOpener-51844aa86315c53347f864258b9b5926-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 51844aa86315c53347f864258b9b5926 2024-11-09T05:40:23,616 INFO [StoreOpener-6c8538a5c08a601164d6d90f17409b33-1 {}] regionserver.HStore(327): Store=6c8538a5c08a601164d6d90f17409b33/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,617 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 6c8538a5c08a601164d6d90f17409b33 2024-11-09T05:40:23,620 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/6c8538a5c08a601164d6d90f17409b33 2024-11-09T05:40:23,620 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/6c8538a5c08a601164d6d90f17409b33 2024-11-09T05:40:23,620 INFO [StoreOpener-51844aa86315c53347f864258b9b5926-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51844aa86315c53347f864258b9b5926 columnFamilyName family 2024-11-09T05:40:23,620 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=e09398052c91,44035,1731130819959, table=test, region=c06857dd8e828d035329855bf73ed41f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T05:40:23,621 DEBUG [StoreOpener-51844aa86315c53347f864258b9b5926-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,621 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/bd305d310e01738dc5cbabbfddb8eb5c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,621 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 6c8538a5c08a601164d6d90f17409b33 2024-11-09T05:40:23,621 INFO [StoreOpener-51844aa86315c53347f864258b9b5926-1 {}] regionserver.HStore(327): Store=51844aa86315c53347f864258b9b5926/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,621 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 6c8538a5c08a601164d6d90f17409b33 2024-11-09T05:40:23,622 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 51844aa86315c53347f864258b9b5926 2024-11-09T05:40:23,622 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened bd305d310e01738dc5cbabbfddb8eb5c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58788861, jitterRate=-0.12397770583629608}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,622 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:23,622 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for bd305d310e01738dc5cbabbfddb8eb5c: Running coprocessor pre-open hook at 1731130823599Writing region info on filesystem at 1731130823599Initializing all the Stores at 1731130823601 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823602 (+1 ms)Cleaning up temporary data from old regions at 1731130823611 (+9 ms)Running coprocessor post-open hooks at 1731130823622 (+11 ms)Region opened successfully at 1731130823622 2024-11-09T05:40:23,623 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/51844aa86315c53347f864258b9b5926 2024-11-09T05:40:23,623 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/51844aa86315c53347f864258b9b5926 2024-11-09T05:40:23,624 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c., pid=55, masterSystemTime=1731130823533 2024-11-09T05:40:23,625 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 51844aa86315c53347f864258b9b5926 2024-11-09T05:40:23,625 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 51844aa86315c53347f864258b9b5926 2024-11-09T05:40:23,626 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=e09398052c91,34331,1731130820076, table=test, region=12126c11ff675bc823b55beab9656286. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T05:40:23,627 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 6c8538a5c08a601164d6d90f17409b33 2024-11-09T05:40:23,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=26 2024-11-09T05:40:23,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 0f1e5e5fd2628827f76a773887689bc8, server=e09398052c91,44035,1731130819959 in 291 msec 2024-11-09T05:40:23,628 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 51844aa86315c53347f864258b9b5926 2024-11-09T05:40:23,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T05:40:23,631 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c. 2024-11-09T05:40:23,631 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c. 2024-11-09T05:40:23,631 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc. 2024-11-09T05:40:23,631 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 42ac5695b6e0e52a5e46661734af96cc, NAME => 'test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-09T05:40:23,632 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/51844aa86315c53347f864258b9b5926/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,632 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:23,632 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,632 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:23,632 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:23,632 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/6c8538a5c08a601164d6d90f17409b33/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,632 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 51844aa86315c53347f864258b9b5926; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73829623, jitterRate=0.10014711320400238}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,633 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 51844aa86315c53347f864258b9b5926 2024-11-09T05:40:23,633 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 51844aa86315c53347f864258b9b5926: Running coprocessor pre-open hook at 1731130823612Writing region info on filesystem at 1731130823612Initializing all the Stores at 1731130823614 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823615 (+1 ms)Cleaning up temporary data from old regions at 1731130823625 (+10 ms)Running coprocessor post-open hooks at 1731130823633 (+8 ms)Region opened successfully at 1731130823633 2024-11-09T05:40:23,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0f1e5e5fd2628827f76a773887689bc8, ASSIGN in 511 msec 2024-11-09T05:40:23,634 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 6c8538a5c08a601164d6d90f17409b33; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60115772, jitterRate=-0.1042051911354065}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,634 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6c8538a5c08a601164d6d90f17409b33 2024-11-09T05:40:23,634 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1731130822466.51844aa86315c53347f864258b9b5926., pid=46, masterSystemTime=1731130823488 2024-11-09T05:40:23,634 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 6c8538a5c08a601164d6d90f17409b33: Running coprocessor pre-open hook at 1731130823608Writing region info on filesystem at 1731130823608Initializing all the Stores at 1731130823610 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823610Cleaning up temporary data from old regions at 1731130823621 (+11 ms)Running coprocessor post-open hooks at 1731130823634 (+13 ms)Region opened successfully at 1731130823634 2024-11-09T05:40:23,635 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,34331,1731130820076 2024-11-09T05:40:23,636 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731130822466.6c8538a5c08a601164d6d90f17409b33., pid=36, masterSystemTime=1731130823439 2024-11-09T05:40:23,636 INFO [StoreOpener-42ac5695b6e0e52a5e46661734af96cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:23,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd305d310e01738dc5cbabbfddb8eb5c, server=e09398052c91,34331,1731130820076 because future has completed 2024-11-09T05:40:23,639 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1731130822466.51844aa86315c53347f864258b9b5926. 2024-11-09T05:40:23,639 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,ddd,1731130822466.51844aa86315c53347f864258b9b5926. 2024-11-09T05:40:23,639 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,nnn,1731130822466.ce006577ade7d81e0bf87827254771f5. 2024-11-09T05:40:23,639 INFO [StoreOpener-42ac5695b6e0e52a5e46661734af96cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42ac5695b6e0e52a5e46661734af96cc columnFamilyName family 2024-11-09T05:40:23,639 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => ce006577ade7d81e0bf87827254771f5, NAME => 'test,nnn,1731130822466.ce006577ade7d81e0bf87827254771f5.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-09T05:40:23,640 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ce006577ade7d81e0bf87827254771f5 2024-11-09T05:40:23,640 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,nnn,1731130822466.ce006577ade7d81e0bf87827254771f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,640 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for ce006577ade7d81e0bf87827254771f5 2024-11-09T05:40:23,640 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:23,640 DEBUG [StoreOpener-42ac5695b6e0e52a5e46661734af96cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,640 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for ce006577ade7d81e0bf87827254771f5 2024-11-09T05:40:23,641 INFO [StoreOpener-42ac5695b6e0e52a5e46661734af96cc-1 {}] regionserver.HStore(327): Store=42ac5695b6e0e52a5e46661734af96cc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,642 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731130822466.6c8538a5c08a601164d6d90f17409b33. 2024-11-09T05:40:23,642 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,mmm,1731130822466.6c8538a5c08a601164d6d90f17409b33. 2024-11-09T05:40:23,642 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:23,642 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,fff,1731130822466.22143fefe98480e8609aa64fb5bbe23e. 2024-11-09T05:40:23,642 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 22143fefe98480e8609aa64fb5bbe23e, NAME => 'test,fff,1731130822466.22143fefe98480e8609aa64fb5bbe23e.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-09T05:40:23,642 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 22143fefe98480e8609aa64fb5bbe23e 2024-11-09T05:40:23,643 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,fff,1731130822466.22143fefe98480e8609aa64fb5bbe23e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,643 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 22143fefe98480e8609aa64fb5bbe23e 2024-11-09T05:40:23,643 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 22143fefe98480e8609aa64fb5bbe23e 2024-11-09T05:40:23,643 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:23,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 056a4a6d72fcadc66bad84500d464f5b, server=e09398052c91,40411,1731130820128 because future has completed 2024-11-09T05:40:23,644 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:23,645 INFO [StoreOpener-ce006577ade7d81e0bf87827254771f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ce006577ade7d81e0bf87827254771f5 2024-11-09T05:40:23,645 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:23,645 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:23,646 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,44035,1731130819959 2024-11-09T05:40:23,648 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:23,649 INFO [StoreOpener-ce006577ade7d81e0bf87827254771f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce006577ade7d81e0bf87827254771f5 columnFamilyName family 2024-11-09T05:40:23,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c8538a5c08a601164d6d90f17409b33, server=e09398052c91,44035,1731130819959 because future has completed 2024-11-09T05:40:23,649 DEBUG [StoreOpener-ce006577ade7d81e0bf87827254771f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,650 INFO [StoreOpener-22143fefe98480e8609aa64fb5bbe23e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 22143fefe98480e8609aa64fb5bbe23e 2024-11-09T05:40:23,651 INFO [StoreOpener-ce006577ade7d81e0bf87827254771f5-1 {}] regionserver.HStore(327): Store=ce006577ade7d81e0bf87827254771f5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,651 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for ce006577ade7d81e0bf87827254771f5 2024-11-09T05:40:23,653 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/ce006577ade7d81e0bf87827254771f5 2024-11-09T05:40:23,653 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/42ac5695b6e0e52a5e46661734af96cc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,653 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/ce006577ade7d81e0bf87827254771f5 2024-11-09T05:40:23,653 INFO [StoreOpener-22143fefe98480e8609aa64fb5bbe23e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22143fefe98480e8609aa64fb5bbe23e columnFamilyName family 2024-11-09T05:40:23,654 DEBUG [StoreOpener-22143fefe98480e8609aa64fb5bbe23e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,654 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 42ac5695b6e0e52a5e46661734af96cc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66314587, jitterRate=-0.011835649609565735}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,654 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:23,654 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for ce006577ade7d81e0bf87827254771f5 2024-11-09T05:40:23,654 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for ce006577ade7d81e0bf87827254771f5 2024-11-09T05:40:23,654 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 42ac5695b6e0e52a5e46661734af96cc: Running coprocessor pre-open hook at 1731130823632Writing region info on filesystem at 1731130823632Initializing all the Stores at 1731130823635 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823635Cleaning up temporary data from old regions at 1731130823645 (+10 ms)Running coprocessor post-open hooks at 1731130823654 (+9 ms)Region opened successfully at 1731130823654 2024-11-09T05:40:23,654 INFO [StoreOpener-22143fefe98480e8609aa64fb5bbe23e-1 {}] regionserver.HStore(327): Store=22143fefe98480e8609aa64fb5bbe23e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,654 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 22143fefe98480e8609aa64fb5bbe23e 2024-11-09T05:40:23,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=5 2024-11-09T05:40:23,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 056a4a6d72fcadc66bad84500d464f5b, server=e09398052c91,40411,1731130820128 in 251 msec 2024-11-09T05:40:23,656 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc., pid=53, masterSystemTime=1731130823533 2024-11-09T05:40:23,656 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/22143fefe98480e8609aa64fb5bbe23e 2024-11-09T05:40:23,657 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/22143fefe98480e8609aa64fb5bbe23e 2024-11-09T05:40:23,657 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 22143fefe98480e8609aa64fb5bbe23e 2024-11-09T05:40:23,658 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 22143fefe98480e8609aa64fb5bbe23e 2024-11-09T05:40:23,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=056a4a6d72fcadc66bad84500d464f5b, ASSIGN in 539 msec 2024-11-09T05:40:23,658 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for ce006577ade7d81e0bf87827254771f5 2024-11-09T05:40:23,659 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc. 2024-11-09T05:40:23,659 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc. 2024-11-09T05:40:23,659 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1. 2024-11-09T05:40:23,659 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => b609b56af9f6fc87d9a153736ec0c8d1, NAME => 'test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-09T05:40:23,660 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:23,660 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,660 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:23,660 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:23,661 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 22143fefe98480e8609aa64fb5bbe23e 2024-11-09T05:40:23,662 INFO [StoreOpener-b609b56af9f6fc87d9a153736ec0c8d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:23,662 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/ce006577ade7d81e0bf87827254771f5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,663 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened ce006577ade7d81e0bf87827254771f5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68870999, jitterRate=0.02625785768032074}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,663 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ce006577ade7d81e0bf87827254771f5 2024-11-09T05:40:23,663 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for ce006577ade7d81e0bf87827254771f5: Running coprocessor pre-open hook at 1731130823641Writing region info on filesystem at 1731130823641Initializing all the Stores at 1731130823643 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823643Cleaning up temporary data from old regions at 1731130823654 (+11 ms)Running coprocessor post-open hooks at 1731130823663 (+9 ms)Region opened successfully at 1731130823663 2024-11-09T05:40:23,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=17 2024-11-09T05:40:23,665 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/22143fefe98480e8609aa64fb5bbe23e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,665 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1731130822466.ce006577ade7d81e0bf87827254771f5., pid=43, masterSystemTime=1731130823488 2024-11-09T05:40:23,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 6c8538a5c08a601164d6d90f17409b33, server=e09398052c91,44035,1731130819959 in 344 msec 2024-11-09T05:40:23,665 INFO [StoreOpener-b609b56af9f6fc87d9a153736ec0c8d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b609b56af9f6fc87d9a153736ec0c8d1 columnFamilyName family 2024-11-09T05:40:23,665 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 22143fefe98480e8609aa64fb5bbe23e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74737824, jitterRate=0.11368036270141602}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,666 DEBUG [StoreOpener-b609b56af9f6fc87d9a153736ec0c8d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,666 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 22143fefe98480e8609aa64fb5bbe23e 2024-11-09T05:40:23,666 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 22143fefe98480e8609aa64fb5bbe23e: Running coprocessor pre-open hook at 1731130823643Writing region info on filesystem at 1731130823643Initializing all the Stores at 1731130823644 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823644Cleaning up temporary data from old regions at 1731130823658 (+14 ms)Running coprocessor post-open hooks at 1731130823666 (+8 ms)Region opened successfully at 1731130823666 2024-11-09T05:40:23,666 INFO [StoreOpener-b609b56af9f6fc87d9a153736ec0c8d1-1 {}] regionserver.HStore(327): Store=b609b56af9f6fc87d9a153736ec0c8d1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,667 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:23,667 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731130822466.22143fefe98480e8609aa64fb5bbe23e., pid=33, masterSystemTime=1731130823439 2024-11-09T05:40:23,668 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:23,668 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:23,669 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1731130822466.ce006577ade7d81e0bf87827254771f5. 2024-11-09T05:40:23,669 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,nnn,1731130822466.ce006577ade7d81e0bf87827254771f5. 2024-11-09T05:40:23,669 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,www,1731130822466.1cc6d417b0bdc1061e6457e90a27badd. 2024-11-09T05:40:23,669 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:23,669 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:23,669 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 1cc6d417b0bdc1061e6457e90a27badd, NAME => 'test,www,1731130822466.1cc6d417b0bdc1061e6457e90a27badd.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-09T05:40:23,670 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1cc6d417b0bdc1061e6457e90a27badd 2024-11-09T05:40:23,670 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,www,1731130822466.1cc6d417b0bdc1061e6457e90a27badd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,670 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 1cc6d417b0bdc1061e6457e90a27badd 2024-11-09T05:40:23,670 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 1cc6d417b0bdc1061e6457e90a27badd 2024-11-09T05:40:23,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6c8538a5c08a601164d6d90f17409b33, ASSIGN in 549 msec 2024-11-09T05:40:23,672 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731130822466.22143fefe98480e8609aa64fb5bbe23e. 2024-11-09T05:40:23,672 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,fff,1731130822466.22143fefe98480e8609aa64fb5bbe23e. 2024-11-09T05:40:23,672 INFO [StoreOpener-1cc6d417b0bdc1061e6457e90a27badd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1cc6d417b0bdc1061e6457e90a27badd 2024-11-09T05:40:23,672 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:23,672 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:23,674 INFO [StoreOpener-1cc6d417b0bdc1061e6457e90a27badd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1cc6d417b0bdc1061e6457e90a27badd columnFamilyName family 2024-11-09T05:40:23,674 DEBUG [StoreOpener-1cc6d417b0bdc1061e6457e90a27badd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,676 INFO [StoreOpener-1cc6d417b0bdc1061e6457e90a27badd-1 {}] regionserver.HStore(327): Store=1cc6d417b0bdc1061e6457e90a27badd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure ce006577ade7d81e0bf87827254771f5, server=e09398052c91,40411,1731130820128 because future has completed 2024-11-09T05:40:23,676 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 1cc6d417b0bdc1061e6457e90a27badd 2024-11-09T05:40:23,677 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/1cc6d417b0bdc1061e6457e90a27badd 2024-11-09T05:40:23,677 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,44035,1731130819959 2024-11-09T05:40:23,678 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/1cc6d417b0bdc1061e6457e90a27badd 2024-11-09T05:40:23,678 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 1cc6d417b0bdc1061e6457e90a27badd 2024-11-09T05:40:23,678 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 1cc6d417b0bdc1061e6457e90a27badd 2024-11-09T05:40:23,679 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/b609b56af9f6fc87d9a153736ec0c8d1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,679 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened b609b56af9f6fc87d9a153736ec0c8d1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61274286, jitterRate=-0.08694198727607727}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,679 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:23,680 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for b609b56af9f6fc87d9a153736ec0c8d1: Running coprocessor pre-open hook at 1731130823660Writing region info on filesystem at 1731130823660Initializing all the Stores at 1731130823661 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823661Cleaning up temporary data from old regions at 1731130823669 (+8 ms)Running coprocessor post-open hooks at 1731130823680 (+11 ms)Region opened successfully at 1731130823680 2024-11-09T05:40:23,681 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22143fefe98480e8609aa64fb5bbe23e, server=e09398052c91,44035,1731130819959 because future has completed 2024-11-09T05:40:23,681 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 1cc6d417b0bdc1061e6457e90a27badd 2024-11-09T05:40:23,681 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1., pid=50, masterSystemTime=1731130823533 2024-11-09T05:40:23,684 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/1cc6d417b0bdc1061e6457e90a27badd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,685 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1. 2024-11-09T05:40:23,685 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1. 2024-11-09T05:40:23,685 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c. 2024-11-09T05:40:23,685 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 8714188f3856e307485ef7426df7ea3c, NAME => 'test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-09T05:40:23,685 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:23,685 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=18 2024-11-09T05:40:23,686 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:23,686 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:23,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure ce006577ade7d81e0bf87827254771f5, server=e09398052c91,40411,1731130820128 in 326 msec 2024-11-09T05:40:23,686 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 1cc6d417b0bdc1061e6457e90a27badd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70778986, jitterRate=0.05468907952308655}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,686 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1cc6d417b0bdc1061e6457e90a27badd 2024-11-09T05:40:23,687 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 1cc6d417b0bdc1061e6457e90a27badd: Running coprocessor pre-open hook at 1731130823670Writing region info on filesystem at 1731130823670Initializing all the Stores at 1731130823671 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823671Cleaning up temporary data from old regions at 1731130823678 (+7 ms)Running coprocessor post-open hooks at 1731130823686 (+8 ms)Region opened successfully at 1731130823686 2024-11-09T05:40:23,688 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731130822466.1cc6d417b0bdc1061e6457e90a27badd., pid=45, masterSystemTime=1731130823488 2024-11-09T05:40:23,688 INFO [StoreOpener-8714188f3856e307485ef7426df7ea3c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:23,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ce006577ade7d81e0bf87827254771f5, ASSIGN in 570 msec 2024-11-09T05:40:23,690 INFO [StoreOpener-8714188f3856e307485ef7426df7ea3c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8714188f3856e307485ef7426df7ea3c columnFamilyName family 2024-11-09T05:40:23,690 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,34331,1731130820076 2024-11-09T05:40:23,690 DEBUG [StoreOpener-8714188f3856e307485ef7426df7ea3c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,691 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731130822466.1cc6d417b0bdc1061e6457e90a27badd. 2024-11-09T05:40:23,691 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,www,1731130822466.1cc6d417b0bdc1061e6457e90a27badd. 2024-11-09T05:40:23,691 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,qqq,1731130822466.8d8a7929b729c2e22434e4a1d52e461a. 2024-11-09T05:40:23,691 INFO [StoreOpener-8714188f3856e307485ef7426df7ea3c-1 {}] regionserver.HStore(327): Store=8714188f3856e307485ef7426df7ea3c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,691 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 8d8a7929b729c2e22434e4a1d52e461a, NAME => 'test,qqq,1731130822466.8d8a7929b729c2e22434e4a1d52e461a.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-09T05:40:23,691 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:23,692 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8d8a7929b729c2e22434e4a1d52e461a 2024-11-09T05:40:23,692 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,qqq,1731130822466.8d8a7929b729c2e22434e4a1d52e461a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,692 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 8d8a7929b729c2e22434e4a1d52e461a 2024-11-09T05:40:23,692 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 8d8a7929b729c2e22434e4a1d52e461a 2024-11-09T05:40:23,692 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:23,693 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:23,694 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:23,694 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:23,695 INFO [StoreOpener-8d8a7929b729c2e22434e4a1d52e461a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8d8a7929b729c2e22434e4a1d52e461a 2024-11-09T05:40:23,695 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure b609b56af9f6fc87d9a153736ec0c8d1, server=e09398052c91,34331,1731130820076 because future has completed 2024-11-09T05:40:23,696 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:23,696 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:23,697 INFO [StoreOpener-8d8a7929b729c2e22434e4a1d52e461a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d8a7929b729c2e22434e4a1d52e461a columnFamilyName family 2024-11-09T05:40:23,697 DEBUG [StoreOpener-8d8a7929b729c2e22434e4a1d52e461a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,698 INFO [StoreOpener-8d8a7929b729c2e22434e4a1d52e461a-1 {}] regionserver.HStore(327): Store=8d8a7929b729c2e22434e4a1d52e461a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,698 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 8d8a7929b729c2e22434e4a1d52e461a 2024-11-09T05:40:23,699 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/8d8a7929b729c2e22434e4a1d52e461a 2024-11-09T05:40:23,700 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/8d8a7929b729c2e22434e4a1d52e461a 2024-11-09T05:40:23,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cc6d417b0bdc1061e6457e90a27badd, server=e09398052c91,40411,1731130820128 because future has completed 2024-11-09T05:40:23,700 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 8d8a7929b729c2e22434e4a1d52e461a 2024-11-09T05:40:23,700 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 8d8a7929b729c2e22434e4a1d52e461a 2024-11-09T05:40:23,702 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/8714188f3856e307485ef7426df7ea3c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,702 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 8714188f3856e307485ef7426df7ea3c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70095461, jitterRate=0.04450376331806183}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,703 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:23,703 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 8d8a7929b729c2e22434e4a1d52e461a 2024-11-09T05:40:23,703 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 8714188f3856e307485ef7426df7ea3c: Running coprocessor pre-open hook at 1731130823686Writing region info on filesystem at 1731130823686Initializing all the Stores at 1731130823687 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823687Cleaning up temporary data from old regions at 1731130823694 (+7 ms)Running coprocessor post-open hooks at 1731130823703 (+9 ms)Region opened successfully at 1731130823703 2024-11-09T05:40:23,705 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c., pid=49, masterSystemTime=1731130823533 2024-11-09T05:40:23,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=29 2024-11-09T05:40:23,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure b609b56af9f6fc87d9a153736ec0c8d1, server=e09398052c91,34331,1731130820076 in 307 msec 2024-11-09T05:40:23,707 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/8d8a7929b729c2e22434e4a1d52e461a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,707 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c. 2024-11-09T05:40:23,707 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 8d8a7929b729c2e22434e4a1d52e461a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71276261, jitterRate=0.06209905445575714}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,707 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c. 2024-11-09T05:40:23,707 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8d8a7929b729c2e22434e4a1d52e461a 2024-11-09T05:40:23,708 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 8d8a7929b729c2e22434e4a1d52e461a: Running coprocessor pre-open hook at 1731130823692Writing region info on filesystem at 1731130823692Initializing all the Stores at 1731130823694 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823694Cleaning up temporary data from old regions at 1731130823700 (+6 ms)Running coprocessor post-open hooks at 1731130823707 (+7 ms)Region opened successfully at 1731130823708 (+1 ms) 2024-11-09T05:40:23,708 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3. 2024-11-09T05:40:23,708 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => b4485a6989fa449d3718b74dd5bef8f3, NAME => 'test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-09T05:40:23,708 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:23,708 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,709 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:23,709 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:23,709 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731130822466.8d8a7929b729c2e22434e4a1d52e461a., pid=44, masterSystemTime=1731130823488 2024-11-09T05:40:23,711 INFO [StoreOpener-b4485a6989fa449d3718b74dd5bef8f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:23,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN in 589 msec 2024-11-09T05:40:23,713 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1731130822466.8d8a7929b729c2e22434e4a1d52e461a. 2024-11-09T05:40:23,713 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,qqq,1731130822466.8d8a7929b729c2e22434e4a1d52e461a. 2024-11-09T05:40:23,713 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,ttt,1731130822466.2339918a4a54d7d8274a43da565fe2a2. 2024-11-09T05:40:23,713 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 2339918a4a54d7d8274a43da565fe2a2, NAME => 'test,ttt,1731130822466.2339918a4a54d7d8274a43da565fe2a2.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-09T05:40:23,713 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2339918a4a54d7d8274a43da565fe2a2 2024-11-09T05:40:23,714 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,ttt,1731130822466.2339918a4a54d7d8274a43da565fe2a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,714 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 2339918a4a54d7d8274a43da565fe2a2 2024-11-09T05:40:23,714 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 2339918a4a54d7d8274a43da565fe2a2 2024-11-09T05:40:23,714 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:23,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d8a7929b729c2e22434e4a1d52e461a, server=e09398052c91,40411,1731130820128 because future has completed 2024-11-09T05:40:23,718 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=8714188f3856e307485ef7426df7ea3c, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,34331,1731130820076 2024-11-09T05:40:23,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8714188f3856e307485ef7426df7ea3c, server=e09398052c91,34331,1731130820076 because future has completed 2024-11-09T05:40:23,724 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=e09398052c91,44035,1731130819959, table=test, region=22143fefe98480e8609aa64fb5bbe23e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T05:40:23,724 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=e09398052c91,44035,1731130819959, table=test, region=c06857dd8e828d035329855bf73ed41f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T05:40:23,725 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=e09398052c91,40411,1731130820128, table=test, region=1cc6d417b0bdc1061e6457e90a27badd. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T05:40:23,726 INFO [StoreOpener-b4485a6989fa449d3718b74dd5bef8f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4485a6989fa449d3718b74dd5bef8f3 columnFamilyName family 2024-11-09T05:40:23,726 DEBUG [StoreOpener-b4485a6989fa449d3718b74dd5bef8f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,727 INFO [StoreOpener-2339918a4a54d7d8274a43da565fe2a2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2339918a4a54d7d8274a43da565fe2a2 2024-11-09T05:40:23,727 INFO [StoreOpener-b4485a6989fa449d3718b74dd5bef8f3-1 {}] regionserver.HStore(327): Store=b4485a6989fa449d3718b74dd5bef8f3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,727 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:23,729 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:23,729 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:23,730 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=e09398052c91,34331,1731130820076, table=test, region=bd305d310e01738dc5cbabbfddb8eb5c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T05:40:23,730 INFO [StoreOpener-2339918a4a54d7d8274a43da565fe2a2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2339918a4a54d7d8274a43da565fe2a2 columnFamilyName family 2024-11-09T05:40:23,730 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=e09398052c91,34331,1731130820076, table=test, region=12126c11ff675bc823b55beab9656286. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T05:40:23,730 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=e09398052c91,34331,1731130820076, table=test, region=8714188f3856e307485ef7426df7ea3c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T05:40:23,730 DEBUG [StoreOpener-2339918a4a54d7d8274a43da565fe2a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=21 2024-11-09T05:40:23,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 8d8a7929b729c2e22434e4a1d52e461a, server=e09398052c91,40411,1731130820128 in 363 msec 2024-11-09T05:40:23,731 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:23,731 INFO [StoreOpener-2339918a4a54d7d8274a43da565fe2a2-1 {}] regionserver.HStore(327): Store=2339918a4a54d7d8274a43da565fe2a2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,731 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:23,731 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 2339918a4a54d7d8274a43da565fe2a2 2024-11-09T05:40:23,732 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/2339918a4a54d7d8274a43da565fe2a2 2024-11-09T05:40:23,733 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/2339918a4a54d7d8274a43da565fe2a2 2024-11-09T05:40:23,733 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:23,733 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 2339918a4a54d7d8274a43da565fe2a2 2024-11-09T05:40:23,734 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 2339918a4a54d7d8274a43da565fe2a2 2024-11-09T05:40:23,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d8a7929b729c2e22434e4a1d52e461a, ASSIGN in 615 msec 2024-11-09T05:40:23,737 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/b4485a6989fa449d3718b74dd5bef8f3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,738 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 2339918a4a54d7d8274a43da565fe2a2 2024-11-09T05:40:23,738 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened b4485a6989fa449d3718b74dd5bef8f3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73261282, jitterRate=0.09167817234992981}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=22 2024-11-09T05:40:23,738 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:23,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 8714188f3856e307485ef7426df7ea3c, server=e09398052c91,34331,1731130820076 in 345 msec 2024-11-09T05:40:23,738 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for b4485a6989fa449d3718b74dd5bef8f3: Running coprocessor pre-open hook at 1731130823709Writing region info on filesystem at 1731130823709Initializing all the Stores at 1731130823711 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823711Cleaning up temporary data from old regions at 1731130823731 (+20 ms)Running coprocessor post-open hooks at 1731130823738 (+7 ms)Region opened successfully at 1731130823738 2024-11-09T05:40:23,740 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3., pid=54, masterSystemTime=1731130823533 2024-11-09T05:40:23,741 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/2339918a4a54d7d8274a43da565fe2a2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,742 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 2339918a4a54d7d8274a43da565fe2a2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70218483, jitterRate=0.046336933970451355}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,742 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2339918a4a54d7d8274a43da565fe2a2 2024-11-09T05:40:23,742 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 2339918a4a54d7d8274a43da565fe2a2: Running coprocessor pre-open hook at 1731130823714Writing region info on filesystem at 1731130823714Initializing all the Stores at 1731130823726 (+12 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823726Cleaning up temporary data from old regions at 1731130823734 (+8 ms)Running coprocessor post-open hooks at 1731130823742 (+8 ms)Region opened successfully at 1731130823742 2024-11-09T05:40:23,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN in 622 msec 2024-11-09T05:40:23,743 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3. 2024-11-09T05:40:23,743 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3. 2024-11-09T05:40:23,744 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0. 2024-11-09T05:40:23,744 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 8d63f643dc4acf4d7c54bcb479b29df0, NAME => 'test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-09T05:40:23,744 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:23,744 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,34331,1731130820076 2024-11-09T05:40:23,744 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,744 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:23,744 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:23,745 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731130822466.2339918a4a54d7d8274a43da565fe2a2., pid=40, masterSystemTime=1731130823488 2024-11-09T05:40:23,747 INFO [StoreOpener-8d63f643dc4acf4d7c54bcb479b29df0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:23,747 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731130822466.2339918a4a54d7d8274a43da565fe2a2. 2024-11-09T05:40:23,747 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,ttt,1731130822466.2339918a4a54d7d8274a43da565fe2a2. 2024-11-09T05:40:23,747 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4485a6989fa449d3718b74dd5bef8f3, server=e09398052c91,34331,1731130820076 because future has completed 2024-11-09T05:40:23,748 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:23,749 INFO [StoreOpener-8d63f643dc4acf4d7c54bcb479b29df0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d63f643dc4acf4d7c54bcb479b29df0 columnFamilyName family 2024-11-09T05:40:23,749 DEBUG [StoreOpener-8d63f643dc4acf4d7c54bcb479b29df0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,750 INFO [StoreOpener-8d63f643dc4acf4d7c54bcb479b29df0-1 {}] regionserver.HStore(327): Store=8d63f643dc4acf4d7c54bcb479b29df0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,750 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:23,751 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:23,751 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2339918a4a54d7d8274a43da565fe2a2, server=e09398052c91,40411,1731130820128 because future has completed 2024-11-09T05:40:23,752 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:23,752 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:23,752 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:23,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=9 2024-11-09T05:40:23,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure b4485a6989fa449d3718b74dd5bef8f3, server=e09398052c91,34331,1731130820076 in 329 msec 2024-11-09T05:40:23,755 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:23,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN in 639 msec 2024-11-09T05:40:23,758 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/8d63f643dc4acf4d7c54bcb479b29df0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,758 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 8d63f643dc4acf4d7c54bcb479b29df0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68344050, jitterRate=0.01840570569038391}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,759 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:23,759 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 8d63f643dc4acf4d7c54bcb479b29df0: Running coprocessor pre-open hook at 1731130823744Writing region info on filesystem at 1731130823744Initializing all the Stores at 1731130823746 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823746Cleaning up temporary data from old regions at 1731130823752 (+6 ms)Running coprocessor post-open hooks at 1731130823759 (+7 ms)Region opened successfully at 1731130823759 2024-11-09T05:40:23,760 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0., pid=48, masterSystemTime=1731130823533 2024-11-09T05:40:23,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=24 2024-11-09T05:40:23,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 2339918a4a54d7d8274a43da565fe2a2, server=e09398052c91,40411,1731130820128 in 419 msec 2024-11-09T05:40:23,762 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0. 2024-11-09T05:40:23,762 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0. 2024-11-09T05:40:23,763 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,iii,1731130822466.348c820a04c95623395fb88f37fad187. 2024-11-09T05:40:23,763 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 348c820a04c95623395fb88f37fad187, NAME => 'test,iii,1731130822466.348c820a04c95623395fb88f37fad187.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-09T05:40:23,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2339918a4a54d7d8274a43da565fe2a2, ASSIGN in 645 msec 2024-11-09T05:40:23,763 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 348c820a04c95623395fb88f37fad187 2024-11-09T05:40:23,763 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,iii,1731130822466.348c820a04c95623395fb88f37fad187.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:23,763 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 348c820a04c95623395fb88f37fad187 2024-11-09T05:40:23,763 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 348c820a04c95623395fb88f37fad187 2024-11-09T05:40:23,764 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,34331,1731130820076 2024-11-09T05:40:23,765 INFO [StoreOpener-348c820a04c95623395fb88f37fad187-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 348c820a04c95623395fb88f37fad187 2024-11-09T05:40:23,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d63f643dc4acf4d7c54bcb479b29df0, server=e09398052c91,34331,1731130820076 because future has completed 2024-11-09T05:40:23,767 INFO [StoreOpener-348c820a04c95623395fb88f37fad187-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 348c820a04c95623395fb88f37fad187 columnFamilyName family 2024-11-09T05:40:23,767 DEBUG [StoreOpener-348c820a04c95623395fb88f37fad187-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:23,768 INFO [StoreOpener-348c820a04c95623395fb88f37fad187-1 {}] regionserver.HStore(327): Store=348c820a04c95623395fb88f37fad187/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:23,768 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 348c820a04c95623395fb88f37fad187 2024-11-09T05:40:23,769 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/348c820a04c95623395fb88f37fad187 2024-11-09T05:40:23,770 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/348c820a04c95623395fb88f37fad187 2024-11-09T05:40:23,770 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 348c820a04c95623395fb88f37fad187 2024-11-09T05:40:23,770 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 348c820a04c95623395fb88f37fad187 2024-11-09T05:40:23,770 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=27 2024-11-09T05:40:23,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 1cc6d417b0bdc1061e6457e90a27badd, server=e09398052c91,40411,1731130820128 in 402 msec 2024-11-09T05:40:23,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1cc6d417b0bdc1061e6457e90a27badd, ASSIGN in 655 msec 2024-11-09T05:40:23,773 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 348c820a04c95623395fb88f37fad187 2024-11-09T05:40:23,776 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/348c820a04c95623395fb88f37fad187/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T05:40:23,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=19 2024-11-09T05:40:23,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 8d63f643dc4acf4d7c54bcb479b29df0, server=e09398052c91,34331,1731130820076 in 388 msec 2024-11-09T05:40:23,777 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 348c820a04c95623395fb88f37fad187; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74095696, jitterRate=0.10411190986633301}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:23,777 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 348c820a04c95623395fb88f37fad187 2024-11-09T05:40:23,777 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 348c820a04c95623395fb88f37fad187: Running coprocessor pre-open hook at 1731130823763Writing region info on filesystem at 1731130823763Initializing all the Stores at 1731130823765 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130823765Cleaning up temporary data from old regions at 1731130823770 (+5 ms)Running coprocessor post-open hooks at 1731130823777 (+7 ms)Region opened successfully at 1731130823777 2024-11-09T05:40:23,778 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731130822466.348c820a04c95623395fb88f37fad187., pid=56, masterSystemTime=1731130823533 2024-11-09T05:40:23,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, ASSIGN in 661 msec 2024-11-09T05:40:23,781 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731130822466.348c820a04c95623395fb88f37fad187. 2024-11-09T05:40:23,781 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,iii,1731130822466.348c820a04c95623395fb88f37fad187. 2024-11-09T05:40:23,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=10 2024-11-09T05:40:23,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 22143fefe98480e8609aa64fb5bbe23e, server=e09398052c91,44035,1731130819959 in 481 msec 2024-11-09T05:40:23,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=22143fefe98480e8609aa64fb5bbe23e, ASSIGN in 666 msec 2024-11-09T05:40:23,785 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=348c820a04c95623395fb88f37fad187, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,34331,1731130820076 2024-11-09T05:40:23,787 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=42ac5695b6e0e52a5e46661734af96cc, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,34331,1731130820076 2024-11-09T05:40:23,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 348c820a04c95623395fb88f37fad187, server=e09398052c91,34331,1731130820076 because future has completed 2024-11-09T05:40:23,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,34331,1731130820076 because future has completed 2024-11-09T05:40:23,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=13 2024-11-09T05:40:23,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 348c820a04c95623395fb88f37fad187, server=e09398052c91,34331,1731130820076 in 360 msec 2024-11-09T05:40:23,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN in 677 msec 2024-11-09T05:40:23,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=7 2024-11-09T05:40:23,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,34331,1731130820076 in 379 msec 2024-11-09T05:40:23,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN in 682 msec 2024-11-09T05:40:23,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=12 2024-11-09T05:40:23,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure bd305d310e01738dc5cbabbfddb8eb5c, server=e09398052c91,34331,1731130820076 in 375 msec 2024-11-09T05:40:23,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN in 688 msec 2024-11-09T05:40:23,806 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=51844aa86315c53347f864258b9b5926, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:23,809 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 51844aa86315c53347f864258b9b5926, server=e09398052c91,40411,1731130820128 because future has completed 2024-11-09T05:40:23,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=16 2024-11-09T05:40:23,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 12126c11ff675bc823b55beab9656286, server=e09398052c91,34331,1731130820076 in 401 msec 2024-11-09T05:40:23,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN in 695 msec 2024-11-09T05:40:23,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=8 2024-11-09T05:40:23,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 51844aa86315c53347f864258b9b5926, server=e09398052c91,40411,1731130820128 in 439 msec 2024-11-09T05:40:23,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=51844aa86315c53347f864258b9b5926, ASSIGN in 701 msec 2024-11-09T05:40:23,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=23 2024-11-09T05:40:23,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure c06857dd8e828d035329855bf73ed41f, server=e09398052c91,44035,1731130819959 in 528 msec 2024-11-09T05:40:23,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c06857dd8e828d035329855bf73ed41f, ASSIGN in 706 msec 2024-11-09T05:40:23,824 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, openSeqNum=2, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:23,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure df09fc3517c167e05c71f65345ca6b80, server=e09398052c91,40411,1731130820128 because future has completed 2024-11-09T05:40:23,829 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=e09398052c91,40411,1731130820128, table=test, region=df09fc3517c167e05c71f65345ca6b80. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T05:40:23,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=11 2024-11-09T05:40:23,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure df09fc3517c167e05c71f65345ca6b80, server=e09398052c91,40411,1731130820128 in 484 msec 2024-11-09T05:40:23,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=4 2024-11-09T05:40:23,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=df09fc3517c167e05c71f65345ca6b80, ASSIGN in 716 msec 2024-11-09T05:40:23,837 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T05:40:23,837 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731130823837"}]},"ts":"1731130823837"} 2024-11-09T05:40:23,840 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-09T05:40:23,841 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T05:40:23,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.3670 sec 2024-11-09T05:40:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T05:40:24,652 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-09T05:40:24,652 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-09T05:40:24,653 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T05:40:24,671 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-09T05:40:24,671 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T05:40:24,672 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-09T05:40:24,673 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T05:40:24,679 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-09T05:40:24,698 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-09T05:40:24,709 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T05:40:24,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T05:40:24,713 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T05:40:24,715 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57800, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T05:40:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 42ac5695b6e0e52a5e46661734af96cc changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b4485a6989fa449d3718b74dd5bef8f3 changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bd305d310e01738dc5cbabbfddb8eb5c changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 348c820a04c95623395fb88f37fad187 changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 12126c11ff675bc823b55beab9656286 changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8d63f643dc4acf4d7c54bcb479b29df0 changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8714188f3856e307485ef7426df7ea3c changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 532f29d232f4841336f315b0406fe4af changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b609b56af9f6fc87d9a153736ec0c8d1 changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 056a4a6d72fcadc66bad84500d464f5b changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 51844aa86315c53347f864258b9b5926 changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region df09fc3517c167e05c71f65345ca6b80 changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 706d9970ba371fdec8f2a079940ca3ae changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ce006577ade7d81e0bf87827254771f5 changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8d8a7929b729c2e22434e4a1d52e461a changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2339918a4a54d7d8274a43da565fe2a2 changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1cc6d417b0bdc1061e6457e90a27badd changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6874567d13af4b499b6eb873decadd10 changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9fee264e99db7cad2e2fb211d7666c11 changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 22143fefe98480e8609aa64fb5bbe23e changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6ea3613715cd8a321124b73881db15ce changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6c8538a5c08a601164d6d90f17409b33 changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d014bc791a8473500fbbfa044123f955 changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c06857dd8e828d035329855bf73ed41f changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0f1e5e5fd2628827f76a773887689bc8 changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d923c8e1af5892d9575aa4b3336f28f9 changed from -1.0 to 0.0, refreshing cache 2024-11-09T05:40:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-09T05:40:24,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-09T05:40:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-09T05:40:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.BalancerClusterState(204): Hosts are {e09398052c91=0} racks are {/default-rack=0} 2024-11-09T05:40:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.BalancerClusterState(303): server 0 has 10 regions 2024-11-09T05:40:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-09T05:40:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.BalancerClusterState(303): server 2 has 8 regions 2024-11-09T05:40:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-09T05:40:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-09T05:40:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-09T05:40:24,766 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-09T05:40:24,766 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-09T05:40:24,766 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-09T05:40:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-09T05:40:24,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-11-09T05:40:24,770 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-11-09T05:40:25,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 451 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-09T05:40:25,221 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-11-09T05:40:25,221 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] master.HMaster(2172): balance hri=8d63f643dc4acf4d7c54bcb479b29df0, source=e09398052c91,34331,1731130820076, destination=e09398052c91,44035,1731130819959 2024-11-09T05:40:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:25,223 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:25,225 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=8d63f643dc4acf4d7c54bcb479b29df0, regionState=CLOSING, regionLocation=e09398052c91,34331,1731130820076 2024-11-09T05:40:25,228 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE because future has completed 2024-11-09T05:40:25,228 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T05:40:25,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8d63f643dc4acf4d7c54bcb479b29df0, server=e09398052c91,34331,1731130820076}] 2024-11-09T05:40:25,386 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close 8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:25,386 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T05:40:25,387 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing 8d63f643dc4acf4d7c54bcb479b29df0, disabling compactions & flushes 2024-11-09T05:40:25,387 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0. 2024-11-09T05:40:25,387 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0. 2024-11-09T05:40:25,387 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0. after waiting 0 ms 2024-11-09T05:40:25,387 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0. 2024-11-09T05:40:25,392 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/8d63f643dc4acf4d7c54bcb479b29df0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:25,395 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0. 2024-11-09T05:40:25,395 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for 8d63f643dc4acf4d7c54bcb479b29df0: Waiting for close lock at 1731130825387Running coprocessor pre-close hooks at 1731130825387Disabling compacts and flushes for region at 1731130825387Disabling writes for close at 1731130825387Writing region close event to WAL at 1731130825388 (+1 ms)Running coprocessor post-close hooks at 1731130825393 (+5 ms)Closed at 1731130825395 (+2 ms) 2024-11-09T05:40:25,395 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding 8d63f643dc4acf4d7c54bcb479b29df0 move to e09398052c91,44035,1731130819959 record at close sequenceid=2 2024-11-09T05:40:25,398 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed 8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:25,399 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=8d63f643dc4acf4d7c54bcb479b29df0, regionState=CLOSED 2024-11-09T05:40:25,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8d63f643dc4acf4d7c54bcb479b29df0, server=e09398052c91,34331,1731130820076 because future has completed 2024-11-09T05:40:25,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-09T05:40:25,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure 8d63f643dc4acf4d7c54bcb479b29df0, server=e09398052c91,34331,1731130820076 in 174 msec 2024-11-09T05:40:25,405 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE; state=CLOSED, location=e09398052c91,44035,1731130819959; forceNewPlan=false, retain=false 2024-11-09T05:40:25,556 INFO [e09398052c91:34809 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-09T05:40:25,556 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPENING, regionLocation=e09398052c91,44035,1731130819959 2024-11-09T05:40:25,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE because future has completed 2024-11-09T05:40:25,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d63f643dc4acf4d7c54bcb479b29df0, server=e09398052c91,44035,1731130819959}] 2024-11-09T05:40:25,718 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0. 2024-11-09T05:40:25,718 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 8d63f643dc4acf4d7c54bcb479b29df0, NAME => 'test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-09T05:40:25,719 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:25,719 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:25,719 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:25,719 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:25,722 INFO [StoreOpener-8d63f643dc4acf4d7c54bcb479b29df0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:25,723 INFO [StoreOpener-8d63f643dc4acf4d7c54bcb479b29df0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d63f643dc4acf4d7c54bcb479b29df0 columnFamilyName family 2024-11-09T05:40:25,723 DEBUG [StoreOpener-8d63f643dc4acf4d7c54bcb479b29df0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:25,723 INFO [StoreOpener-8d63f643dc4acf4d7c54bcb479b29df0-1 {}] regionserver.HStore(327): Store=8d63f643dc4acf4d7c54bcb479b29df0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:25,724 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:25,724 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:25,726 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:25,726 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:25,726 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:25,728 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:25,729 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 8d63f643dc4acf4d7c54bcb479b29df0; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68234986, jitterRate=0.01678052544593811}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:25,730 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:25,730 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 8d63f643dc4acf4d7c54bcb479b29df0: Running coprocessor pre-open hook at 1731130825719Writing region info on filesystem at 1731130825719Initializing all the Stores at 1731130825721 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130825721Cleaning up temporary data from old regions at 1731130825726 (+5 ms)Running coprocessor post-open hooks at 1731130825730 (+4 ms)Region opened successfully at 1731130825730 2024-11-09T05:40:25,731 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0., pid=59, masterSystemTime=1731130825713 2024-11-09T05:40:25,733 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0. 2024-11-09T05:40:25,734 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0. 2024-11-09T05:40:25,734 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, openSeqNum=5, regionLocation=e09398052c91,44035,1731130819959 2024-11-09T05:40:25,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d63f643dc4acf4d7c54bcb479b29df0, server=e09398052c91,44035,1731130819959 because future has completed 2024-11-09T05:40:25,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-11-09T05:40:25,741 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure 8d63f643dc4acf4d7c54bcb479b29df0, server=e09398052c91,44035,1731130819959 in 178 msec 2024-11-09T05:40:25,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE in 520 msec 2024-11-09T05:40:25,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-09T05:40:25,826 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T05:40:25,835 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-09T05:40:25,852 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=378, OpenFileDescriptor=623, MaxFileDescriptor=1048576, SystemLoadAverage=296, ProcessCount=11, AvailableMemoryMB=3496 2024-11-09T05:40:25,855 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-09T05:40:25,856 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-09T05:40:25,856 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing e09398052c91,34331,1731130820076 2024-11-09T05:40:25,857 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server e09398052c91,34331,1731130820076: Simulated kill ***** 2024-11-09T05:40:25,857 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-09T05:40:25,859 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-09T05:40:25,861 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-09T05:40:25,862 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-09T05:40:25,862 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-09T05:40:25,862 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 321666192 }, "NonHeapMemoryUsage": { "committed": 153944064, "init": 7667712, "max": -1, "used": 151396520 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "e09398052c91" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "e09398052c91" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "e09398052c91" } ] } 2024-11-09T05:40:25,865 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34809 {}] master.MasterRpcServices(700): e09398052c91,34331,1731130820076 reported a fatal error: ***** ABORTING region server e09398052c91,34331,1731130820076: Simulated kill ***** 2024-11-09T05:40:25,867 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e09398052c91,34331,1731130820076' ***** 2024-11-09T05:40:25,867 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-09T05:40:25,867 INFO [RS:1;e09398052c91:34331 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T05:40:25,867 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T05:40:25,867 INFO [RS:1;e09398052c91:34331 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-09T05:40:25,867 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T05:40:25,868 INFO [RS:1;e09398052c91:34331 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-09T05:40:25,868 INFO [RS:1;e09398052c91:34331 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T05:40:25,868 INFO [RS:1;e09398052c91:34331 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;e09398052c91:34331. 2024-11-09T05:40:25,868 DEBUG [RS:1;e09398052c91:34331 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T05:40:25,873 DEBUG [RS:1;e09398052c91:34331 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T05:40:25,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741836_1012 (size=33749) 2024-11-09T05:40:25,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741836_1012 (size=33749) 2024-11-09T05:40:25,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741833_1009 (size=2774) 2024-11-09T05:40:25,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741833_1009 (size=2774) 2024-11-09T05:40:25,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741833_1009 (size=2774) 2024-11-09T05:40:25,884 DEBUG [RS:1;e09398052c91:34331 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T05:40:25,884 INFO [RS:1;e09398052c91:34331 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T05:40:25,885 INFO [RS:1;e09398052c91:34331 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34331 2024-11-09T05:40:25,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34331-0x100fb3de9450002, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e09398052c91,34331,1731130820076 2024-11-09T05:40:25,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T05:40:25,892 INFO [RS:1;e09398052c91:34331 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T05:40:25,893 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e09398052c91,34331,1731130820076] 2024-11-09T05:40:25,895 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e09398052c91,34331,1731130820076 already deleted, retry=false 2024-11-09T05:40:25,896 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of e09398052c91,34331,1731130820076 on e09398052c91,34809,1731130819107 2024-11-09T05:40:25,900 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:25,905 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:25,905 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=60 for e09398052c91,34331,1731130820076 (carryingMeta=true) e09398052c91,34331,1731130820076/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1541dc12[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-09T05:40:25,906 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true, isMeta: true 2024-11-09T05:40:25,908 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting 2024-11-09T05:40:25,910 INFO [PEWorker-1 {}] master.SplitWALManager(105): e09398052c91,34331,1731130820076 WAL count=1, meta=true 2024-11-09T05:40:25,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta}] 2024-11-09T05:40:25,918 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:25,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959}] 2024-11-09T05:40:25,942 INFO [regionserver/e09398052c91:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T05:40:25,969 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T05:40:25,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34331-0x100fb3de9450002, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T05:40:25,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34331-0x100fb3de9450002, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T05:40:25,996 INFO [RS:1;e09398052c91:34331 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T05:40:25,996 INFO [RS:1;e09398052c91:34331 {}] regionserver.HRegionServer(1031): Exiting; stopping=e09398052c91,34331,1731130820076; zookeeper connection closed. 2024-11-09T05:40:25,996 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@329b3c30 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@329b3c30 2024-11-09T05:40:26,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44035 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-11-09T05:40:26,109 INFO [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, size=33.0 K (33749bytes) 2024-11-09T05:40:26,110 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-09T05:40:26,110 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-09T05:40:26,110 INFO [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta 2024-11-09T05:40:26,117 INFO [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta after 4ms 2024-11-09T05:40:26,123 INFO [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta took 15ms 2024-11-09T05:40:26,133 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-09T05:40:26,140 INFO [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta.temp 2024-11-09T05:40:26,144 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta so closing down 2024-11-09T05:40:26,144 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-09T05:40:26,148 INFO [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta.temp 2024-11-09T05:40:26,153 INFO [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-11-09T05:40:26,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741864_1040 (size=33508) 2024-11-09T05:40:26,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741864_1040 (size=33508) 2024-11-09T05:40:26,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741864_1040 (size=33508) 2024-11-09T05:40:26,171 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta.temp (wrote 61 edits, skipped 0 edits in 3 ms) 2024-11-09T05:40:26,174 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta.temp to hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-09T05:40:26,177 INFO [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 49 ms; skipped=1; WAL=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-11-09T05:40:26,177 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, journal: Splitting hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, size=33.0 K (33749bytes) at 1731130826109Finishing writing output for hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta so closing down at 1731130826144 (+35 ms)Creating recovered edits writer path=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta.temp at 1731130826148 (+4 ms)3 split writer threads finished at 1731130826153 (+5 ms)Closed recovered edits writer path=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta.temp (wrote 61 edits, skipped 0 edits in 3 ms) at 1731130826171 (+18 ms)Rename recovered edits hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta.temp to hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1731130826174 (+3 ms)Processed 62 edits across 1 Regions in 49 ms; skipped=1; WAL=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1731130826177 (+3 ms) 2024-11-09T05:40:26,178 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta 2024-11-09T05:40:26,179 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-09T05:40:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-11-09T05:40:26,188 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta to hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/oldWALs 2024-11-09T05:40:26,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-09T05:40:26,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 in 268 msec 2024-11-09T05:40:26,192 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:26,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-09T05:40:26,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 in 280 msec 2024-11-09T05:40:26,196 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting dir is empty, no logs to split. 2024-11-09T05:40:26,196 INFO [PEWorker-1 {}] master.SplitWALManager(105): e09398052c91,34331,1731130820076 WAL count=0, meta=true 2024-11-09T05:40:26,196 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if e09398052c91,34331,1731130820076 WAL splitting is done? wals=0, meta=true 2024-11-09T05:40:26,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-09T05:40:26,199 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:26,200 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T05:40:26,350 DEBUG [e09398052c91:34809 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-09T05:40:26,351 DEBUG [e09398052c91:34809 {}] balancer.BalancerClusterState(204): Hosts are {e09398052c91=0} racks are {/default-rack=0} 2024-11-09T05:40:26,351 DEBUG [e09398052c91:34809 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-09T05:40:26,351 DEBUG [e09398052c91:34809 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-09T05:40:26,351 DEBUG [e09398052c91:34809 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-09T05:40:26,351 DEBUG [e09398052c91:34809 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-09T05:40:26,351 INFO [e09398052c91:34809 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-09T05:40:26,351 INFO [e09398052c91:34809 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-09T05:40:26,351 DEBUG [e09398052c91:34809 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-09T05:40:26,352 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:26,353 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPENING 2024-11-09T05:40:26,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:26,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:26,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:26,356 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:26,356 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:26,356 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:26,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128}] 2024-11-09T05:40:26,356 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:26,513 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-09T05:40:26,514 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T05:40:26,514 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-09T05:40:26,516 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e09398052c91%2C40411%2C1731130820128.meta, suffix=.meta, logDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,40411,1731130820128, archiveDir=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/oldWALs, maxLogs=32 2024-11-09T05:40:26,532 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,40411,1731130820128/e09398052c91%2C40411%2C1731130820128.meta.1731130826517.meta, exclude list is [], retry=0 2024-11-09T05:40:26,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34201,DS-8282eccd-3379-4ce2-9612-3e9a0257084e,DISK] 2024-11-09T05:40:26,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43869,DS-ed00c975-41ab-41f9-90d6-754aff306015,DISK] 2024-11-09T05:40:26,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39285,DS-0dfa7937-2ca7-4bc6-8d88-b6076a7fadac,DISK] 2024-11-09T05:40:26,538 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,40411,1731130820128/e09398052c91%2C40411%2C1731130820128.meta.1731130826517.meta 2024-11-09T05:40:26,538 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44645:44645),(127.0.0.1/127.0.0.1:46203:46203),(127.0.0.1/127.0.0.1:42387:42387)] 2024-11-09T05:40:26,538 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-09T05:40:26,539 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-09T05:40:26,539 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-09T05:40:26,539 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-09T05:40:26,539 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-09T05:40:26,539 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:26,539 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-09T05:40:26,539 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-09T05:40:26,541 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T05:40:26,542 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T05:40:26,542 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:26,543 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T05:40:26,543 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T05:40:26,544 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T05:40:26,544 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:26,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T05:40:26,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T05:40:26,545 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T05:40:26,546 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:26,546 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T05:40:26,546 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T05:40:26,547 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T05:40:26,547 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:26,547 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T05:40:26,547 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T05:40:26,548 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740 2024-11-09T05:40:26,550 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740 2024-11-09T05:40:26,551 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-09T05:40:26,567 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-09T05:40:26,571 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-11-09T05:40:26,576 INFO [regionserver/e09398052c91:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-09T05:40:26,577 INFO [regionserver/e09398052c91:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-09T05:40:26,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741841_1017 (size=45) 2024-11-09T05:40:26,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741842_1018 (size=45) 2024-11-09T05:40:26,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741831_1007 (size=1321) 2024-11-09T05:40:26,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741840_1016 (size=42) 2024-11-09T05:40:26,643 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/.tmp/info/4f6c3cc37e974f6fbdcb8626bd387f79 is 138, key is test,bbb,1731130822466.9fee264e99db7cad2e2fb211d7666c11./info:regioninfo/1731130823495/Put/seqid=0 2024-11-09T05:40:26,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741866_1042 (size=36382) 2024-11-09T05:40:26,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741866_1042 (size=36382) 2024-11-09T05:40:26,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741866_1042 (size=36382) 2024-11-09T05:40:26,659 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/.tmp/info/4f6c3cc37e974f6fbdcb8626bd387f79 2024-11-09T05:40:26,721 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/.tmp/ns/76398e8a837b42559d28d5ca810ff989 is 43, key is default/ns:d/1731130822238/Put/seqid=0 2024-11-09T05:40:26,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741867_1043 (size=5153) 2024-11-09T05:40:26,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741867_1043 (size=5153) 2024-11-09T05:40:26,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741867_1043 (size=5153) 2024-11-09T05:40:26,733 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/.tmp/ns/76398e8a837b42559d28d5ca810ff989 2024-11-09T05:40:26,758 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/.tmp/table/98f234f0fecd4d1aa1fb272d043442c5 is 40, key is test/table:state/1731130823837/Put/seqid=0 2024-11-09T05:40:26,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741868_1044 (size=5165) 2024-11-09T05:40:26,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741868_1044 (size=5165) 2024-11-09T05:40:26,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741868_1044 (size=5165) 2024-11-09T05:40:26,767 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/.tmp/table/98f234f0fecd4d1aa1fb272d043442c5 2024-11-09T05:40:26,776 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/.tmp/info/4f6c3cc37e974f6fbdcb8626bd387f79 as hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/info/4f6c3cc37e974f6fbdcb8626bd387f79 2024-11-09T05:40:26,784 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/info/4f6c3cc37e974f6fbdcb8626bd387f79, entries=273, sequenceid=64, filesize=35.5 K 2024-11-09T05:40:26,786 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/.tmp/ns/76398e8a837b42559d28d5ca810ff989 as hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/ns/76398e8a837b42559d28d5ca810ff989 2024-11-09T05:40:26,795 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/ns/76398e8a837b42559d28d5ca810ff989, entries=2, sequenceid=64, filesize=5.0 K 2024-11-09T05:40:26,797 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/.tmp/table/98f234f0fecd4d1aa1fb272d043442c5 as hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/table/98f234f0fecd4d1aa1fb272d043442c5 2024-11-09T05:40:26,806 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/table/98f234f0fecd4d1aa1fb272d043442c5, entries=2, sequenceid=64, filesize=5.0 K 2024-11-09T05:40:26,810 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 239ms, sequenceid=64, compaction requested=false; wal=null 2024-11-09T05:40:26,811 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-09T05:40:26,814 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-09T05:40:26,815 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T05:40:26,816 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T05:40:26,817 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T05:40:26,818 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T05:40:26,821 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-11-09T05:40:26,822 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68019817, jitterRate=0.013574257493019104}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T05:40:26,822 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-09T05:40:26,823 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731130826539Writing region info on filesystem at 1731130826539Initializing all the Stores at 1731130826541 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731130826541Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731130826541Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130826541Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731130826541Obtaining lock to block concurrent updates at 1731130826571 (+30 ms)Preparing flush snapshotting stores in 1588230740 at 1731130826571Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1731130826573 (+2 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731130826574 (+1 ms)Flushing 1588230740/info: creating writer at 1731130826575 (+1 ms)Flushing 1588230740/info: appending metadata at 1731130826633 (+58 ms)Flushing 1588230740/info: closing flushed file at 1731130826637 (+4 ms)Flushing 1588230740/ns: creating writer at 1731130826704 (+67 ms)Flushing 1588230740/ns: appending metadata at 1731130826720 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731130826720Flushing 1588230740/table: creating writer at 1731130826741 (+21 ms)Flushing 1588230740/table: appending metadata at 1731130826757 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731130826757Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38d3bfac: reopening flushed file at 1731130826775 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68464080: reopening flushed file at 1731130826785 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bf602d3: reopening flushed file at 1731130826795 (+10 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 239ms, sequenceid=64, compaction requested=false; wal=null at 1731130826810 (+15 ms)Cleaning up temporary data from old regions at 1731130826816 (+6 ms)Running coprocessor post-open hooks at 1731130826822 (+6 ms)Region opened successfully at 1731130826822 2024-11-09T05:40:26,824 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1731130826508 2024-11-09T05:40:26,827 DEBUG [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-09T05:40:26,827 INFO [RS_OPEN_META-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-09T05:40:26,828 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:26,828 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:26,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:26,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:26,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:26,830 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:26,830 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:26,830 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:26,831 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:26,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-09T05:40:26,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 in 475 msec 2024-11-09T05:40:26,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-11-09T05:40:26,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 636 msec 2024-11-09T05:40:26,837 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): e09398052c91,34331,1731130820076 had 9 regions 2024-11-09T05:40:26,838 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true, isMeta: false 2024-11-09T05:40:26,840 INFO [PEWorker-1 {}] master.SplitWALManager(105): e09398052c91,34331,1731130820076 WAL count=1, meta=false 2024-11-09T05:40:26,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693}] 2024-11-09T05:40:26,842 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:26,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959}] 2024-11-09T05:40:26,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44035 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-11-09T05:40:27,013 INFO [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.1731130821693, size=2.7 K (2774bytes) 2024-11-09T05:40:27,013 INFO [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.1731130821693 2024-11-09T05:40:27,013 INFO [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.1731130821693 after 0ms 2024-11-09T05:40:27,017 INFO [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.1731130821693 took 5ms 2024-11-09T05:40:27,020 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 532f29d232f4841336f315b0406fe4af: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T05:40:27,022 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 12126c11ff675bc823b55beab9656286: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T05:40:27,024 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for bd305d310e01738dc5cbabbfddb8eb5c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T05:40:27,026 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 42ac5695b6e0e52a5e46661734af96cc: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T05:40:27,027 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for b609b56af9f6fc87d9a153736ec0c8d1: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T05:40:27,029 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 8714188f3856e307485ef7426df7ea3c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T05:40:27,030 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for b4485a6989fa449d3718b74dd5bef8f3: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T05:40:27,032 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 8d63f643dc4acf4d7c54bcb479b29df0: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T05:40:27,033 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 348c820a04c95623395fb88f37fad187: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T05:40:27,033 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.1731130821693 so closing down 2024-11-09T05:40:27,033 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-09T05:40:27,033 INFO [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-11-09T05:40:27,034 INFO [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 17 ms; skipped=10; WAL=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.1731130821693, size=2.7 K, length=2774, corrupted=false, cancelled=false 2024-11-09T05:40:27,034 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.1731130821693, journal: Splitting hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.1731130821693, size=2.7 K (2774bytes) at 1731130827013Finishing writing output for hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.1731130821693 so closing down at 1731130827033 (+20 ms)3 split writer threads finished at 1731130827033Processed 10 edits across 0 Regions in 17 ms; skipped=10; WAL=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.1731130821693, size=2.7 K, length=2774, corrupted=false, cancelled=false at 1731130827034 (+1 ms) 2024-11-09T05:40:27,034 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.1731130821693 2024-11-09T05:40:27,034 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e09398052c91:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-09T05:40:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34809 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-11-09T05:40:27,039 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting/e09398052c91%2C34331%2C1731130820076.1731130821693 to hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/oldWALs 2024-11-09T05:40:27,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-09T05:40:27,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 in 196 msec 2024-11-09T05:40:27,043 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-11-09T05:40:27,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 in 203 msec 2024-11-09T05:40:27,047 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/WALs/e09398052c91,34331,1731130820076-splitting dir is empty, no logs to split. 2024-11-09T05:40:27,047 INFO [PEWorker-1 {}] master.SplitWALManager(105): e09398052c91,34331,1731130820076 WAL count=0, meta=false 2024-11-09T05:40:27,047 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if e09398052c91,34331,1731130820076 WAL splitting is done? wals=0, meta=false 2024-11-09T05:40:27,049 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN}] 2024-11-09T05:40:27,051 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN 2024-11-09T05:40:27,052 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T05:40:27,053 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:27,054 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T05:40:27,054 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,055 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T05:40:27,056 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,057 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T05:40:27,057 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,058 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T05:40:27,059 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,061 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T05:40:27,062 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,062 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T05:40:27,063 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,064 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T05:40:27,202 DEBUG [e09398052c91:34809 {}] balancer.BalancerClusterState(204): Hosts are {e09398052c91=0} racks are {/default-rack=0} 2024-11-09T05:40:27,203 DEBUG [e09398052c91:34809 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-09T05:40:27,203 DEBUG [e09398052c91:34809 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-09T05:40:27,203 DEBUG [e09398052c91:34809 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-09T05:40:27,203 DEBUG [e09398052c91:34809 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-09T05:40:27,203 INFO [e09398052c91:34809 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-09T05:40:27,203 INFO [e09398052c91:34809 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-09T05:40:27,203 DEBUG [e09398052c91:34809 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-09T05:40:27,203 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=532f29d232f4841336f315b0406fe4af, regionState=OPENING, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,204 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=12126c11ff675bc823b55beab9656286, regionState=OPENING, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,204 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,205 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, regionLocation=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,204 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server e09398052c91:34331 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e09398052c91/172.17.0.2:34331 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T05:40:27,205 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, regionLocation=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,205 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, regionLocation=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,206 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=348c820a04c95623395fb88f37fad187, regionState=OPENING, regionLocation=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e09398052c91,34331,1731130820076, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=e09398052c91,34331,1731130820076, seqNum=-1, error=java.net.ConnectException: Call to address=e09398052c91:34331 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e09398052c91/172.17.0.2:34331 2024-11-09T05:40:27,206 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, regionLocation=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=e09398052c91,34331,1731130820076, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-09T05:40:27,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=e09398052c91,34331,1731130820076, seqNum=-1 from cache 2024-11-09T05:40:27,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address e09398052c91:34331 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e09398052c91/172.17.0.2:34331 2024-11-09T05:40:27,208 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server e09398052c91:34331 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e09398052c91/172.17.0.2:34331 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T05:40:27,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e09398052c91,34331,1731130820076, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=e09398052c91:34331 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e09398052c91/172.17.0.2:34331 2024-11-09T05:40:27,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e09398052c91,34331,1731130820076, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=e09398052c91:34331 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e09398052c91/172.17.0.2:34331 2024-11-09T05:40:27,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e09398052c91,34331,1731130820076, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=e09398052c91:34331 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e09398052c91/172.17.0.2:34331 2024-11-09T05:40:27,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e09398052c91,34331,1731130820076, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=e09398052c91:34331 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e09398052c91/172.17.0.2:34331 2024-11-09T05:40:27,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e09398052c91,34331,1731130820076, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=e09398052c91:34331 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e09398052c91/172.17.0.2:34331 2024-11-09T05:40:27,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e09398052c91,34331,1731130820076, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=e09398052c91:34331 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e09398052c91/172.17.0.2:34331 2024-11-09T05:40:27,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=e09398052c91,34331,1731130820076, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=e09398052c91:34331 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e09398052c91/172.17.0.2:34331 2024-11-09T05:40:27,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address e09398052c91:34331 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: e09398052c91/172.17.0.2:34331 2024-11-09T05:40:27,310 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T05:40:27,310 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e09398052c91,40411,1731130820128, seqNum=-1] 2024-11-09T05:40:27,310 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T05:40:27,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44569, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T05:40:27,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN because future has completed 2024-11-09T05:40:27,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 532f29d232f4841336f315b0406fe4af, server=e09398052c91,40411,1731130820128}] 2024-11-09T05:40:27,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN because future has completed 2024-11-09T05:40:27,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 348c820a04c95623395fb88f37fad187, server=e09398052c91,44035,1731130819959}] 2024-11-09T05:40:27,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN because future has completed 2024-11-09T05:40:27,325 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN because future has completed 2024-11-09T05:40:27,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN because future has completed 2024-11-09T05:40:27,327 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN because future has completed 2024-11-09T05:40:27,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd305d310e01738dc5cbabbfddb8eb5c, server=e09398052c91,40411,1731130820128}] 2024-11-09T05:40:27,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN because future has completed 2024-11-09T05:40:27,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN because future has completed 2024-11-09T05:40:27,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8714188f3856e307485ef7426df7ea3c, server=e09398052c91,44035,1731130819959}] 2024-11-09T05:40:27,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure b609b56af9f6fc87d9a153736ec0c8d1, server=e09398052c91,44035,1731130819959}] 2024-11-09T05:40:27,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128}] 2024-11-09T05:40:27,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12126c11ff675bc823b55beab9656286, server=e09398052c91,40411,1731130820128}] 2024-11-09T05:40:27,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4485a6989fa449d3718b74dd5bef8f3, server=e09398052c91,44035,1731130819959}] 2024-11-09T05:40:27,471 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc. 2024-11-09T05:40:27,472 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 42ac5695b6e0e52a5e46661734af96cc, NAME => 'test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-09T05:40:27,472 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:27,472 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:27,472 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:27,472 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:27,474 INFO [StoreOpener-42ac5695b6e0e52a5e46661734af96cc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:27,475 INFO [StoreOpener-42ac5695b6e0e52a5e46661734af96cc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42ac5695b6e0e52a5e46661734af96cc columnFamilyName family 2024-11-09T05:40:27,475 DEBUG [StoreOpener-42ac5695b6e0e52a5e46661734af96cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:27,475 INFO [StoreOpener-42ac5695b6e0e52a5e46661734af96cc-1 {}] regionserver.HStore(327): Store=42ac5695b6e0e52a5e46661734af96cc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:27,476 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:27,476 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:27,478 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:27,478 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:27,478 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:27,479 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3. 2024-11-09T05:40:27,479 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => b4485a6989fa449d3718b74dd5bef8f3, NAME => 'test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-09T05:40:27,480 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:27,480 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:27,480 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:27,480 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:27,480 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:27,481 INFO [StoreOpener-b4485a6989fa449d3718b74dd5bef8f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:27,482 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 42ac5695b6e0e52a5e46661734af96cc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62168979, jitterRate=-0.07361002266407013}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:27,482 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:27,482 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 42ac5695b6e0e52a5e46661734af96cc: Running coprocessor pre-open hook at 1731130827472Writing region info on filesystem at 1731130827472Initializing all the Stores at 1731130827473 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130827473Cleaning up temporary data from old regions at 1731130827478 (+5 ms)Running coprocessor post-open hooks at 1731130827482 (+4 ms)Region opened successfully at 1731130827482 2024-11-09T05:40:27,482 INFO [StoreOpener-b4485a6989fa449d3718b74dd5bef8f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4485a6989fa449d3718b74dd5bef8f3 columnFamilyName family 2024-11-09T05:40:27,482 DEBUG [StoreOpener-b4485a6989fa449d3718b74dd5bef8f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:27,483 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc., pid=80, masterSystemTime=1731130827466 2024-11-09T05:40:27,483 INFO [StoreOpener-b4485a6989fa449d3718b74dd5bef8f3-1 {}] regionserver.HStore(327): Store=b4485a6989fa449d3718b74dd5bef8f3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:27,483 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:27,484 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:27,485 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc. 2024-11-09T05:40:27,485 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc. 2024-11-09T05:40:27,485 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,lll,1731130822466.12126c11ff675bc823b55beab9656286. 2024-11-09T05:40:27,486 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 12126c11ff675bc823b55beab9656286, NAME => 'test,lll,1731130822466.12126c11ff675bc823b55beab9656286.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-09T05:40:27,486 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:27,486 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 12126c11ff675bc823b55beab9656286 2024-11-09T05:40:27,486 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,lll,1731130822466.12126c11ff675bc823b55beab9656286.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:27,486 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:27,486 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 12126c11ff675bc823b55beab9656286 2024-11-09T05:40:27,486 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:27,486 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 12126c11ff675bc823b55beab9656286 2024-11-09T05:40:27,488 INFO [StoreOpener-12126c11ff675bc823b55beab9656286-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 12126c11ff675bc823b55beab9656286 2024-11-09T05:40:27,489 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:27,489 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-09T05:40:27,489 INFO [StoreOpener-12126c11ff675bc823b55beab9656286-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12126c11ff675bc823b55beab9656286 columnFamilyName family 2024-11-09T05:40:27,490 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,490 DEBUG [StoreOpener-12126c11ff675bc823b55beab9656286-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:27,490 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened b4485a6989fa449d3718b74dd5bef8f3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62071505, jitterRate=-0.07506249845027924}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:27,490 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:27,490 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for b4485a6989fa449d3718b74dd5bef8f3: Running coprocessor pre-open hook at 1731130827480Writing region info on filesystem at 1731130827480Initializing all the Stores at 1731130827481 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130827481Cleaning up temporary data from old regions at 1731130827486 (+5 ms)Running coprocessor post-open hooks at 1731130827490 (+4 ms)Region opened successfully at 1731130827490 2024-11-09T05:40:27,490 INFO [StoreOpener-12126c11ff675bc823b55beab9656286-1 {}] regionserver.HStore(327): Store=12126c11ff675bc823b55beab9656286/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:27,491 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 12126c11ff675bc823b55beab9656286 2024-11-09T05:40:27,491 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3., pid=82, masterSystemTime=1731130827475 2024-11-09T05:40:27,490 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,492 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/12126c11ff675bc823b55beab9656286 2024-11-09T05:40:27,493 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3. 2024-11-09T05:40:27,493 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(153): Opened test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3. 2024-11-09T05:40:27,493 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1. 2024-11-09T05:40:27,493 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => b609b56af9f6fc87d9a153736ec0c8d1, NAME => 'test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-09T05:40:27,493 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:27,494 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/12126c11ff675bc823b55beab9656286 2024-11-09T05:40:27,494 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:27,494 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:27,494 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:27,494 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 12126c11ff675bc823b55beab9656286 2024-11-09T05:40:27,494 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 12126c11ff675bc823b55beab9656286 2024-11-09T05:40:27,495 INFO [StoreOpener-b609b56af9f6fc87d9a153736ec0c8d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:27,496 INFO [StoreOpener-b609b56af9f6fc87d9a153736ec0c8d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b609b56af9f6fc87d9a153736ec0c8d1 columnFamilyName family 2024-11-09T05:40:27,496 DEBUG [StoreOpener-b609b56af9f6fc87d9a153736ec0c8d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:27,497 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 12126c11ff675bc823b55beab9656286 2024-11-09T05:40:27,497 INFO [StoreOpener-b609b56af9f6fc87d9a153736ec0c8d1-1 {}] regionserver.HStore(327): Store=b609b56af9f6fc87d9a153736ec0c8d1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:27,497 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:27,498 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:27,498 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 12126c11ff675bc823b55beab9656286; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74384360, jitterRate=0.10841333866119385}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:27,498 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 12126c11ff675bc823b55beab9656286 2024-11-09T05:40:27,498 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 12126c11ff675bc823b55beab9656286: Running coprocessor pre-open hook at 1731130827486Writing region info on filesystem at 1731130827486Initializing all the Stores at 1731130827488 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130827488Cleaning up temporary data from old regions at 1731130827494 (+6 ms)Running coprocessor post-open hooks at 1731130827498 (+4 ms)Region opened successfully at 1731130827498 2024-11-09T05:40:27,499 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731130822466.12126c11ff675bc823b55beab9656286., pid=81, masterSystemTime=1731130827466 2024-11-09T05:40:27,500 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:27,501 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:27,501 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:27,501 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731130822466.12126c11ff675bc823b55beab9656286. 2024-11-09T05:40:27,500 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 4 more 2024-11-09T05:40:27,502 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened test,lll,1731130822466.12126c11ff675bc823b55beab9656286. 2024-11-09T05:40:27,502 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af. 2024-11-09T05:40:27,502 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 532f29d232f4841336f315b0406fe4af, NAME => 'test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-09T05:40:27,502 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:27,502 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4485a6989fa449d3718b74dd5bef8f3, server=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,502 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:27,502 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:27,503 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:27,503 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:27,503 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:27,504 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4485a6989fa449d3718b74dd5bef8f3, server=e09398052c91,44035,1731130819959 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,505 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened b609b56af9f6fc87d9a153736ec0c8d1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60241703, jitterRate=-0.10232867300510406}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:27,505 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:27,505 INFO [StoreOpener-532f29d232f4841336f315b0406fe4af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:27,505 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for b609b56af9f6fc87d9a153736ec0c8d1: Running coprocessor pre-open hook at 1731130827494Writing region info on filesystem at 1731130827494Initializing all the Stores at 1731130827495 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130827495Cleaning up temporary data from old regions at 1731130827501 (+6 ms)Running coprocessor post-open hooks at 1731130827505 (+4 ms)Region opened successfully at 1731130827505 2024-11-09T05:40:27,506 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1., pid=79, masterSystemTime=1731130827475 2024-11-09T05:40:27,506 INFO [StoreOpener-532f29d232f4841336f315b0406fe4af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 532f29d232f4841336f315b0406fe4af columnFamilyName family 2024-11-09T05:40:27,506 DEBUG [StoreOpener-532f29d232f4841336f315b0406fe4af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:27,507 INFO [StoreOpener-532f29d232f4841336f315b0406fe4af-1 {}] regionserver.HStore(327): Store=532f29d232f4841336f315b0406fe4af/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:27,507 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:27,507 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:27,508 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1. 2024-11-09T05:40:27,508 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1. 2024-11-09T05:40:27,508 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,iii,1731130822466.348c820a04c95623395fb88f37fad187. 2024-11-09T05:40:27,508 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 348c820a04c95623395fb88f37fad187, NAME => 'test,iii,1731130822466.348c820a04c95623395fb88f37fad187.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-09T05:40:27,509 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 348c820a04c95623395fb88f37fad187 2024-11-09T05:40:27,509 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:27,509 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,iii,1731130822466.348c820a04c95623395fb88f37fad187.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:27,509 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 348c820a04c95623395fb88f37fad187 2024-11-09T05:40:27,509 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 348c820a04c95623395fb88f37fad187 2024-11-09T05:40:27,509 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:27,509 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:27,511 INFO [StoreOpener-348c820a04c95623395fb88f37fad187-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 348c820a04c95623395fb88f37fad187 2024-11-09T05:40:27,512 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:27,512 INFO [StoreOpener-348c820a04c95623395fb88f37fad187-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 348c820a04c95623395fb88f37fad187 columnFamilyName family 2024-11-09T05:40:27,512 DEBUG [StoreOpener-348c820a04c95623395fb88f37fad187-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:27,512 INFO [StoreOpener-348c820a04c95623395fb88f37fad187-1 {}] regionserver.HStore(327): Store=348c820a04c95623395fb88f37fad187/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:27,513 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 348c820a04c95623395fb88f37fad187 2024-11-09T05:40:27,513 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 532f29d232f4841336f315b0406fe4af; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66262512, jitterRate=-0.012611627578735352}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:27,513 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:27,513 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 532f29d232f4841336f315b0406fe4af: Running coprocessor pre-open hook at 1731130827503Writing region info on filesystem at 1731130827503Initializing all the Stores at 1731130827505 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130827505Cleaning up temporary data from old regions at 1731130827509 (+4 ms)Running coprocessor post-open hooks at 1731130827513 (+4 ms)Region opened successfully at 1731130827513 2024-11-09T05:40:27,513 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/348c820a04c95623395fb88f37fad187 2024-11-09T05:40:27,514 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af., pid=75, masterSystemTime=1731130827466 2024-11-09T05:40:27,515 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/348c820a04c95623395fb88f37fad187 2024-11-09T05:40:27,515 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 348c820a04c95623395fb88f37fad187 2024-11-09T05:40:27,515 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 348c820a04c95623395fb88f37fad187 2024-11-09T05:40:27,516 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af. 2024-11-09T05:40:27,516 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af. 2024-11-09T05:40:27,517 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c. 2024-11-09T05:40:27,517 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => bd305d310e01738dc5cbabbfddb8eb5c, NAME => 'test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-09T05:40:27,517 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:27,517 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:27,517 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 348c820a04c95623395fb88f37fad187 2024-11-09T05:40:27,517 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:27,517 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:27,518 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 348c820a04c95623395fb88f37fad187; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65223913, jitterRate=-0.028087958693504333}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:27,518 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 348c820a04c95623395fb88f37fad187 2024-11-09T05:40:27,518 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 348c820a04c95623395fb88f37fad187: Running coprocessor pre-open hook at 1731130827509Writing region info on filesystem at 1731130827509Initializing all the Stores at 1731130827510 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130827510Cleaning up temporary data from old regions at 1731130827515 (+5 ms)Running coprocessor post-open hooks at 1731130827518 (+3 ms)Region opened successfully at 1731130827518 2024-11-09T05:40:27,519 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731130822466.348c820a04c95623395fb88f37fad187., pid=76, masterSystemTime=1731130827475 2024-11-09T05:40:27,519 INFO [StoreOpener-bd305d310e01738dc5cbabbfddb8eb5c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:27,520 INFO [StoreOpener-bd305d310e01738dc5cbabbfddb8eb5c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd305d310e01738dc5cbabbfddb8eb5c columnFamilyName family 2024-11-09T05:40:27,520 DEBUG [StoreOpener-bd305d310e01738dc5cbabbfddb8eb5c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:27,521 INFO [StoreOpener-bd305d310e01738dc5cbabbfddb8eb5c-1 {}] regionserver.HStore(327): Store=bd305d310e01738dc5cbabbfddb8eb5c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:27,521 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:27,522 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731130822466.348c820a04c95623395fb88f37fad187. 2024-11-09T05:40:27,522 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,iii,1731130822466.348c820a04c95623395fb88f37fad187. 2024-11-09T05:40:27,522 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:27,522 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c. 2024-11-09T05:40:27,522 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 8714188f3856e307485ef7426df7ea3c, NAME => 'test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-09T05:40:27,523 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:27,523 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T05:40:27,523 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:27,523 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:27,524 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:27,524 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:27,524 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:27,525 INFO [StoreOpener-8714188f3856e307485ef7426df7ea3c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:27,526 INFO [StoreOpener-8714188f3856e307485ef7426df7ea3c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8714188f3856e307485ef7426df7ea3c columnFamilyName family 2024-11-09T05:40:27,526 DEBUG [StoreOpener-8714188f3856e307485ef7426df7ea3c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T05:40:27,526 INFO [StoreOpener-8714188f3856e307485ef7426df7ea3c-1 {}] regionserver.HStore(327): Store=8714188f3856e307485ef7426df7ea3c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T05:40:27,527 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:27,527 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:27,527 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:27,528 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened bd305d310e01738dc5cbabbfddb8eb5c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71418682, jitterRate=0.06422129273414612}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:27,528 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:27,528 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for bd305d310e01738dc5cbabbfddb8eb5c: Running coprocessor pre-open hook at 1731130827517Writing region info on filesystem at 1731130827517Initializing all the Stores at 1731130827518 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130827518Cleaning up temporary data from old regions at 1731130827525 (+7 ms)Running coprocessor post-open hooks at 1731130827528 (+3 ms)Region opened successfully at 1731130827528 2024-11-09T05:40:27,529 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:27,529 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c., pid=77, masterSystemTime=1731130827466 2024-11-09T05:40:27,529 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:27,529 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:27,532 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c. 2024-11-09T05:40:27,532 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c. 2024-11-09T05:40:27,532 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:27,533 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 8714188f3856e307485ef7426df7ea3c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66324547, jitterRate=-0.011687234044075012}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T05:40:27,533 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:27,533 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 8714188f3856e307485ef7426df7ea3c: Running coprocessor pre-open hook at 1731130827523Writing region info on filesystem at 1731130827523Initializing all the Stores at 1731130827524 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731130827524Cleaning up temporary data from old regions at 1731130827529 (+5 ms)Running coprocessor post-open hooks at 1731130827533 (+4 ms)Region opened successfully at 1731130827533 2024-11-09T05:40:27,534 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c., pid=78, masterSystemTime=1731130827475 2024-11-09T05:40:27,536 DEBUG [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c. 2024-11-09T05:40:27,536 INFO [RS_OPEN_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c. 2024-11-09T05:40:27,573 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:27,573 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,573 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:27,574 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:27,574 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,575 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:27,575 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:27,575 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:27,575 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:27,576 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:27,580 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:27,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,581 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,582 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:27,584 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:27,584 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:27,590 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:27,590 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,590 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,590 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:27,590 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,590 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,590 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,590 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,590 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,590 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,590 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,590 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,590 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,590 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:27,590 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN 2024-11-09T05:40:27,590 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 532f29d232f4841336f315b0406fe4af, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,590 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 348c820a04c95623395fb88f37fad187, server=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,590 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd305d310e01738dc5cbabbfddb8eb5c, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,590 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8714188f3856e307485ef7426df7ea3c, server=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,590 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure b609b56af9f6fc87d9a153736ec0c8d1, server=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,590 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,591 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12126c11ff675bc823b55beab9656286, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,591 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4485a6989fa449d3718b74dd5bef8f3, server=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,591 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,591 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,591 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,591 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,591 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,591 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,591 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,591 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,591 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,591 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,591 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,591 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,591 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,591 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,591 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,591 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,591 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,592 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,592 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,592 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,592 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:27,592 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,592 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN 2024-11-09T05:40:27,592 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 532f29d232f4841336f315b0406fe4af, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,592 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 348c820a04c95623395fb88f37fad187, server=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,592 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd305d310e01738dc5cbabbfddb8eb5c, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,592 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8714188f3856e307485ef7426df7ea3c, server=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,592 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure b609b56af9f6fc87d9a153736ec0c8d1, server=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,592 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,592 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12126c11ff675bc823b55beab9656286, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,592 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4485a6989fa449d3718b74dd5bef8f3, server=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,593 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 16 msec 2024-11-09T05:40:27,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:27,595 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:27,595 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:27,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,597 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,597 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,597 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,598 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN to state=OFFLINE, location=null, table=test, region=b4485a6989fa449d3718b74dd5bef8f3 to restore RIT 2024-11-09T05:40:27,598 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN to state=OFFLINE, location=null, table=test, region=b609b56af9f6fc87d9a153736ec0c8d1 to restore RIT 2024-11-09T05:40:27,598 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN to state=OFFLINE, location=null, table=test, region=532f29d232f4841336f315b0406fe4af to restore RIT 2024-11-09T05:40:27,598 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN to state=OFFLINE, location=null, table=test, region=bd305d310e01738dc5cbabbfddb8eb5c to restore RIT 2024-11-09T05:40:27,598 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN to state=OFFLINE, location=null, table=test, region=8714188f3856e307485ef7426df7ea3c to restore RIT 2024-11-09T05:40:27,598 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN to state=OFFLINE, location=null, table=test, region=42ac5695b6e0e52a5e46661734af96cc to restore RIT 2024-11-09T05:40:27,598 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN to state=OFFLINE, location=null, table=test, region=348c820a04c95623395fb88f37fad187 to restore RIT 2024-11-09T05:40:27,598 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN to state=OFFLINE, location=null, table=test, region=12126c11ff675bc823b55beab9656286 to restore RIT 2024-11-09T05:40:27,599 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130857599 2024-11-09T05:40:27,599 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:27,599 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,601 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:27,602 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:27,602 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12126c11ff675bc823b55beab9656286, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,603 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12126c11ff675bc823b55beab9656286, server=e09398052c91,40411,1731130820128 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,622 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T05:40:27,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,625 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:27,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,630 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,632 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:27,632 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130887632 2024-11-09T05:40:27,632 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130947632 2024-11-09T05:40:27,632 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 32 msec 2024-11-09T05:40:27,632 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:27,632 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,632 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:27,632 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:27,633 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:27,633 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:27,633 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:27,633 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:27,634 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:27,640 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:27,640 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,640 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,640 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,640 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,640 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,640 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,640 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,640 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,640 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,640 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,640 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,640 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,640 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,640 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,640 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,641 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,641 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,641 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,641 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,641 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,641 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,641 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:27,642 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:27,643 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:27,646 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:27,646 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,646 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,646 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:27,646 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 532f29d232f4841336f315b0406fe4af, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 348c820a04c95623395fb88f37fad187, server=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd305d310e01738dc5cbabbfddb8eb5c, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8714188f3856e307485ef7426df7ea3c, server=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure b609b56af9f6fc87d9a153736ec0c8d1, server=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12126c11ff675bc823b55beab9656286, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,647 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,647 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,648 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,648 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,648 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,648 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,648 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,648 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,648 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,648 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,648 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,648 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,648 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,648 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:27,648 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,648 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN 2024-11-09T05:40:27,648 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 532f29d232f4841336f315b0406fe4af, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,648 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 348c820a04c95623395fb88f37fad187, server=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,648 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd305d310e01738dc5cbabbfddb8eb5c, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,648 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8714188f3856e307485ef7426df7ea3c, server=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,648 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure b609b56af9f6fc87d9a153736ec0c8d1, server=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,648 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,648 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12126c11ff675bc823b55beab9656286, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,648 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-11-09T05:40:27,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:27,650 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:27,650 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:27,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,652 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,652 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,652 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,653 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN to state=OFFLINE, location=null, table=test, region=b4485a6989fa449d3718b74dd5bef8f3 to restore RIT 2024-11-09T05:40:27,653 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN to state=OFFLINE, location=null, table=test, region=b609b56af9f6fc87d9a153736ec0c8d1 to restore RIT 2024-11-09T05:40:27,654 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN to state=OFFLINE, location=null, table=test, region=532f29d232f4841336f315b0406fe4af to restore RIT 2024-11-09T05:40:27,654 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN to state=OFFLINE, location=null, table=test, region=bd305d310e01738dc5cbabbfddb8eb5c to restore RIT 2024-11-09T05:40:27,654 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN to state=OFFLINE, location=null, table=test, region=8714188f3856e307485ef7426df7ea3c to restore RIT 2024-11-09T05:40:27,654 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN to state=OFFLINE, location=null, table=test, region=42ac5695b6e0e52a5e46661734af96cc to restore RIT 2024-11-09T05:40:27,654 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN to state=OFFLINE, location=null, table=test, region=348c820a04c95623395fb88f37fad187 to restore RIT 2024-11-09T05:40:27,654 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN to state=OFFLINE, location=null, table=test, region=12126c11ff675bc823b55beab9656286 to restore RIT 2024-11-09T05:40:27,655 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130857655 2024-11-09T05:40:27,655 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:27,655 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,656 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:27,658 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:27,658 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,658 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,666 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,666 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,668 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:27,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,669 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,671 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,671 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:27,671 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130887671 2024-11-09T05:40:27,671 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130947671 2024-11-09T05:40:27,671 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-11-09T05:40:27,671 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:27,671 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,671 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:27,671 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:27,671 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:27,671 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:27,671 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:27,671 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:27,672 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:27,675 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:27,675 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,676 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,676 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,677 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,677 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,677 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:27,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:27,678 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:27,681 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:27,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:27,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:27,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN 2024-11-09T05:40:27,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 532f29d232f4841336f315b0406fe4af, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 348c820a04c95623395fb88f37fad187, server=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,683 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-09T05:40:27,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd305d310e01738dc5cbabbfddb8eb5c, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8714188f3856e307485ef7426df7ea3c, server=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure b609b56af9f6fc87d9a153736ec0c8d1, server=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,683 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,684 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,684 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,684 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,684 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,684 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,684 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,684 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,684 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,684 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,684 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,684 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,684 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,684 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,684 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,684 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,684 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,684 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,684 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,684 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,684 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,684 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:27,684 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,684 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN 2024-11-09T05:40:27,684 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 532f29d232f4841336f315b0406fe4af, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,684 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 348c820a04c95623395fb88f37fad187, server=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,684 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd305d310e01738dc5cbabbfddb8eb5c, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,684 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8714188f3856e307485ef7426df7ea3c, server=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,685 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure b609b56af9f6fc87d9a153736ec0c8d1, server=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,685 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=72, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,685 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-11-09T05:40:27,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:27,686 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:27,686 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:27,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,689 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,689 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,689 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,690 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN to state=OFFLINE, location=null, table=test, region=b4485a6989fa449d3718b74dd5bef8f3 to restore RIT 2024-11-09T05:40:27,690 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN to state=OFFLINE, location=null, table=test, region=b609b56af9f6fc87d9a153736ec0c8d1 to restore RIT 2024-11-09T05:40:27,690 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN to state=OFFLINE, location=null, table=test, region=532f29d232f4841336f315b0406fe4af to restore RIT 2024-11-09T05:40:27,690 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN to state=OFFLINE, location=null, table=test, region=bd305d310e01738dc5cbabbfddb8eb5c to restore RIT 2024-11-09T05:40:27,690 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN to state=OFFLINE, location=null, table=test, region=8714188f3856e307485ef7426df7ea3c to restore RIT 2024-11-09T05:40:27,690 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN to state=OFFLINE, location=null, table=test, region=42ac5695b6e0e52a5e46661734af96cc to restore RIT 2024-11-09T05:40:27,690 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN to state=OFFLINE, location=null, table=test, region=348c820a04c95623395fb88f37fad187 to restore RIT 2024-11-09T05:40:27,690 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN to state=OFFLINE, location=null, table=test, region=12126c11ff675bc823b55beab9656286 to restore RIT 2024-11-09T05:40:27,691 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130857691 2024-11-09T05:40:27,691 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:27,691 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,692 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:27,693 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:27,693 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure b609b56af9f6fc87d9a153736ec0c8d1, server=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,693 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure b609b56af9f6fc87d9a153736ec0c8d1, server=e09398052c91,44035,1731130819959 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,703 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:27,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,706 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:27,706 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130887706 2024-11-09T05:40:27,706 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130947706 2024-11-09T05:40:27,706 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-11-09T05:40:27,706 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:27,706 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,706 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:27,707 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:27,707 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:27,707 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:27,707 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:27,707 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:27,708 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:27,713 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:27,713 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,713 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,713 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,713 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,713 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,713 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,713 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,713 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,713 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,713 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,713 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,713 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,713 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,713 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,713 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,713 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,713 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,713 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,713 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:27,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:27,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:27,718 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:27,718 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,718 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,718 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:27,718 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,718 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,718 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,718 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,718 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 532f29d232f4841336f315b0406fe4af, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 348c820a04c95623395fb88f37fad187, server=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd305d310e01738dc5cbabbfddb8eb5c, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8714188f3856e307485ef7426df7ea3c, server=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure b609b56af9f6fc87d9a153736ec0c8d1, server=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,719 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,719 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,719 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,719 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,719 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,720 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,720 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,720 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,720 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,720 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:27,720 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,720 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN 2024-11-09T05:40:27,720 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 532f29d232f4841336f315b0406fe4af, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,720 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 348c820a04c95623395fb88f37fad187, server=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,720 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd305d310e01738dc5cbabbfddb8eb5c, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,720 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8714188f3856e307485ef7426df7ea3c, server=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,720 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure b609b56af9f6fc87d9a153736ec0c8d1, server=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,720 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-09T05:40:27,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:27,721 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:27,721 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:27,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,723 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,723 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,723 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,724 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN to state=OFFLINE, location=null, table=test, region=b4485a6989fa449d3718b74dd5bef8f3 to restore RIT 2024-11-09T05:40:27,725 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN to state=OFFLINE, location=null, table=test, region=b609b56af9f6fc87d9a153736ec0c8d1 to restore RIT 2024-11-09T05:40:27,725 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN to state=OFFLINE, location=null, table=test, region=532f29d232f4841336f315b0406fe4af to restore RIT 2024-11-09T05:40:27,725 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN to state=OFFLINE, location=null, table=test, region=bd305d310e01738dc5cbabbfddb8eb5c to restore RIT 2024-11-09T05:40:27,725 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN to state=OFFLINE, location=null, table=test, region=8714188f3856e307485ef7426df7ea3c to restore RIT 2024-11-09T05:40:27,725 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN to state=OFFLINE, location=null, table=test, region=42ac5695b6e0e52a5e46661734af96cc to restore RIT 2024-11-09T05:40:27,725 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN to state=OFFLINE, location=null, table=test, region=348c820a04c95623395fb88f37fad187 to restore RIT 2024-11-09T05:40:27,725 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN to state=OFFLINE, location=null, table=test, region=12126c11ff675bc823b55beab9656286 to restore RIT 2024-11-09T05:40:27,725 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130857725 2024-11-09T05:40:27,725 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:27,725 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,727 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:27,728 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:27,728 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8714188f3856e307485ef7426df7ea3c, server=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,728 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,728 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8714188f3856e307485ef7426df7ea3c, server=e09398052c91,44035,1731130819959 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:27,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,739 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:27,739 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130887739 2024-11-09T05:40:27,739 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130947739 2024-11-09T05:40:27,739 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-09T05:40:27,740 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:27,740 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,740 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:27,740 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:27,740 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:27,740 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:27,740 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:27,741 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:27,741 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:27,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:27,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,746 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,746 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:27,747 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:27,748 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:27,750 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:27,750 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,750 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 532f29d232f4841336f315b0406fe4af, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 348c820a04c95623395fb88f37fad187, server=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd305d310e01738dc5cbabbfddb8eb5c, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8714188f3856e307485ef7426df7ea3c, server=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,751 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,751 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,752 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,752 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,752 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,752 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,752 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,752 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,752 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,752 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,752 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,752 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,752 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,752 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:27,752 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,752 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN 2024-11-09T05:40:27,752 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 532f29d232f4841336f315b0406fe4af, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,752 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 348c820a04c95623395fb88f37fad187, server=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,752 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd305d310e01738dc5cbabbfddb8eb5c, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,752 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8714188f3856e307485ef7426df7ea3c, server=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,752 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-09T05:40:27,753 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:27,754 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:27,754 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:27,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,756 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,756 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,756 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,757 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN to state=OFFLINE, location=null, table=test, region=b4485a6989fa449d3718b74dd5bef8f3 to restore RIT 2024-11-09T05:40:27,757 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN to state=OFFLINE, location=null, table=test, region=b609b56af9f6fc87d9a153736ec0c8d1 to restore RIT 2024-11-09T05:40:27,757 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN to state=OFFLINE, location=null, table=test, region=532f29d232f4841336f315b0406fe4af to restore RIT 2024-11-09T05:40:27,757 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN to state=OFFLINE, location=null, table=test, region=bd305d310e01738dc5cbabbfddb8eb5c to restore RIT 2024-11-09T05:40:27,757 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN to state=OFFLINE, location=null, table=test, region=8714188f3856e307485ef7426df7ea3c to restore RIT 2024-11-09T05:40:27,757 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN to state=OFFLINE, location=null, table=test, region=42ac5695b6e0e52a5e46661734af96cc to restore RIT 2024-11-09T05:40:27,757 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN to state=OFFLINE, location=null, table=test, region=348c820a04c95623395fb88f37fad187 to restore RIT 2024-11-09T05:40:27,757 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN to state=OFFLINE, location=null, table=test, region=12126c11ff675bc823b55beab9656286 to restore RIT 2024-11-09T05:40:27,757 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130857757 2024-11-09T05:40:27,758 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:27,758 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,759 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:27,761 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:27,761 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd305d310e01738dc5cbabbfddb8eb5c, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,761 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd305d310e01738dc5cbabbfddb8eb5c, server=e09398052c91,40411,1731130820128 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,764 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,764 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,764 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:27,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,767 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:27,767 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130887767 2024-11-09T05:40:27,767 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130947767 2024-11-09T05:40:27,767 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-09T05:40:27,767 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:27,767 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,767 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:27,768 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:27,768 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:27,768 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:27,768 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:27,768 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:27,769 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:27,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:27,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,771 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,771 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:27,772 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:27,772 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:27,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 532f29d232f4841336f315b0406fe4af, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 348c820a04c95623395fb88f37fad187, server=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd305d310e01738dc5cbabbfddb8eb5c, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,774 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,774 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,774 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,774 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,774 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,774 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,774 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,774 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 532f29d232f4841336f315b0406fe4af, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 348c820a04c95623395fb88f37fad187, server=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd305d310e01738dc5cbabbfddb8eb5c, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,774 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-09T05:40:27,775 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:27,775 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:27,775 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:27,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,777 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,777 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,777 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN to state=OFFLINE, location=null, table=test, region=b4485a6989fa449d3718b74dd5bef8f3 to restore RIT 2024-11-09T05:40:27,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN to state=OFFLINE, location=null, table=test, region=b609b56af9f6fc87d9a153736ec0c8d1 to restore RIT 2024-11-09T05:40:27,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN to state=OFFLINE, location=null, table=test, region=532f29d232f4841336f315b0406fe4af to restore RIT 2024-11-09T05:40:27,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN to state=OFFLINE, location=null, table=test, region=bd305d310e01738dc5cbabbfddb8eb5c to restore RIT 2024-11-09T05:40:27,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN to state=OFFLINE, location=null, table=test, region=8714188f3856e307485ef7426df7ea3c to restore RIT 2024-11-09T05:40:27,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN to state=OFFLINE, location=null, table=test, region=42ac5695b6e0e52a5e46661734af96cc to restore RIT 2024-11-09T05:40:27,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN to state=OFFLINE, location=null, table=test, region=348c820a04c95623395fb88f37fad187 to restore RIT 2024-11-09T05:40:27,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN to state=OFFLINE, location=null, table=test, region=12126c11ff675bc823b55beab9656286 to restore RIT 2024-11-09T05:40:27,779 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130857779 2024-11-09T05:40:27,779 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:27,779 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,780 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:27,781 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:27,781 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 348c820a04c95623395fb88f37fad187, server=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,781 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 348c820a04c95623395fb88f37fad187, server=e09398052c91,44035,1731130819959 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,785 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,786 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:27,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,788 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:27,788 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130887788 2024-11-09T05:40:27,788 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130947788 2024-11-09T05:40:27,788 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-09T05:40:27,788 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:27,788 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,788 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:27,788 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:27,788 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:27,788 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:27,788 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:27,789 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:27,789 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:27,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:27,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:27,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:27,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 532f29d232f4841336f315b0406fe4af, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 348c820a04c95623395fb88f37fad187, server=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,794 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,794 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,794 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,794 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,794 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,794 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,794 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,795 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:27,795 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,795 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN 2024-11-09T05:40:27,795 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 532f29d232f4841336f315b0406fe4af, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,795 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure 348c820a04c95623395fb88f37fad187, server=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,795 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-09T05:40:27,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:27,796 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:27,796 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:27,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,799 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,799 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,799 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,800 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN to state=OFFLINE, location=null, table=test, region=b4485a6989fa449d3718b74dd5bef8f3 to restore RIT 2024-11-09T05:40:27,800 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN to state=OFFLINE, location=null, table=test, region=b609b56af9f6fc87d9a153736ec0c8d1 to restore RIT 2024-11-09T05:40:27,800 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN to state=OFFLINE, location=null, table=test, region=532f29d232f4841336f315b0406fe4af to restore RIT 2024-11-09T05:40:27,800 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN to state=OFFLINE, location=null, table=test, region=bd305d310e01738dc5cbabbfddb8eb5c to restore RIT 2024-11-09T05:40:27,800 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN to state=OFFLINE, location=null, table=test, region=8714188f3856e307485ef7426df7ea3c to restore RIT 2024-11-09T05:40:27,800 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN to state=OFFLINE, location=null, table=test, region=42ac5695b6e0e52a5e46661734af96cc to restore RIT 2024-11-09T05:40:27,800 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN to state=OFFLINE, location=null, table=test, region=348c820a04c95623395fb88f37fad187 to restore RIT 2024-11-09T05:40:27,801 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN to state=OFFLINE, location=null, table=test, region=12126c11ff675bc823b55beab9656286 to restore RIT 2024-11-09T05:40:27,801 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130857801 2024-11-09T05:40:27,801 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:27,801 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,802 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:27,803 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:27,803 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 532f29d232f4841336f315b0406fe4af, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,804 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 532f29d232f4841336f315b0406fe4af, server=e09398052c91,40411,1731130820128 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:27,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,809 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:27,809 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130887809 2024-11-09T05:40:27,809 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130947809 2024-11-09T05:40:27,809 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-09T05:40:27,809 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:27,810 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,810 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:27,810 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:27,810 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:27,810 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:27,810 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:27,810 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:27,811 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:27,812 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:27,813 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,813 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,813 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,813 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,813 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,813 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,813 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,813 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,813 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,813 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,813 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,813 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,813 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,813 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,813 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,813 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:27,814 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:27,814 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 532f29d232f4841336f315b0406fe4af, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,816 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,816 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,816 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,816 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,816 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,816 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,816 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,816 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN 2024-11-09T05:40:27,816 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 532f29d232f4841336f315b0406fe4af, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,817 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-09T05:40:27,817 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:27,817 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:27,817 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:27,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,819 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,819 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,819 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,820 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN to state=OFFLINE, location=null, table=test, region=b4485a6989fa449d3718b74dd5bef8f3 to restore RIT 2024-11-09T05:40:27,820 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN to state=OFFLINE, location=null, table=test, region=b609b56af9f6fc87d9a153736ec0c8d1 to restore RIT 2024-11-09T05:40:27,820 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN to state=OFFLINE, location=null, table=test, region=532f29d232f4841336f315b0406fe4af to restore RIT 2024-11-09T05:40:27,820 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN to state=OFFLINE, location=null, table=test, region=bd305d310e01738dc5cbabbfddb8eb5c to restore RIT 2024-11-09T05:40:27,820 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN to state=OFFLINE, location=null, table=test, region=8714188f3856e307485ef7426df7ea3c to restore RIT 2024-11-09T05:40:27,821 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN to state=OFFLINE, location=null, table=test, region=42ac5695b6e0e52a5e46661734af96cc to restore RIT 2024-11-09T05:40:27,821 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN to state=OFFLINE, location=null, table=test, region=348c820a04c95623395fb88f37fad187 to restore RIT 2024-11-09T05:40:27,821 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN to state=OFFLINE, location=null, table=test, region=12126c11ff675bc823b55beab9656286 to restore RIT 2024-11-09T05:40:27,821 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130857821 2024-11-09T05:40:27,821 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:27,821 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,823 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:27,824 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:27,824 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN 2024-11-09T05:40:27,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,824 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,826 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,826 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:27,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,830 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,831 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,831 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:27,831 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130887831 2024-11-09T05:40:27,831 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130947831 2024-11-09T05:40:27,831 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-09T05:40:27,831 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:27,831 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,832 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:27,832 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:27,832 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:27,832 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:27,832 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:27,832 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:27,833 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:27,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:27,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,836 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,836 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:27,837 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:27,837 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:27,838 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:27,838 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,839 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,839 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:27,839 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,839 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,839 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,839 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,839 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,839 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,839 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,839 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,839 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,839 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:27,839 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN 2024-11-09T05:40:27,839 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,839 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,839 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,839 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,839 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,839 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,839 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,839 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,839 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,839 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,839 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,840 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,840 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,840 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,840 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,840 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,840 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,840 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,840 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,840 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,840 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:27,840 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,840 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN 2024-11-09T05:40:27,840 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-09T05:40:27,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:27,841 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:27,841 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:27,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,843 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,843 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,843 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,844 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN to state=OFFLINE, location=null, table=test, region=b4485a6989fa449d3718b74dd5bef8f3 to restore RIT 2024-11-09T05:40:27,844 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN to state=OFFLINE, location=null, table=test, region=b609b56af9f6fc87d9a153736ec0c8d1 to restore RIT 2024-11-09T05:40:27,844 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN to state=OFFLINE, location=null, table=test, region=532f29d232f4841336f315b0406fe4af to restore RIT 2024-11-09T05:40:27,844 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN to state=OFFLINE, location=null, table=test, region=bd305d310e01738dc5cbabbfddb8eb5c to restore RIT 2024-11-09T05:40:27,844 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN to state=OFFLINE, location=null, table=test, region=8714188f3856e307485ef7426df7ea3c to restore RIT 2024-11-09T05:40:27,845 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN to state=OFFLINE, location=null, table=test, region=42ac5695b6e0e52a5e46661734af96cc to restore RIT 2024-11-09T05:40:27,845 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN to state=OFFLINE, location=null, table=test, region=348c820a04c95623395fb88f37fad187 to restore RIT 2024-11-09T05:40:27,845 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=12126c11ff675bc823b55beab9656286, ASSIGN to state=OFFLINE, location=null, table=test, region=12126c11ff675bc823b55beab9656286 to restore RIT 2024-11-09T05:40:27,845 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130857845 2024-11-09T05:40:27,845 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:27,845 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,846 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:27,847 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:27,847 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:27,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,847 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,850 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,851 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:27,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,855 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:27,855 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130887855 2024-11-09T05:40:27,855 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130947855 2024-11-09T05:40:27,855 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-09T05:40:27,855 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:27,855 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,855 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:27,855 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:27,855 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:27,855 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:27,855 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:27,856 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:27,856 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:27,858 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:27,858 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,858 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,858 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,858 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,858 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,858 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,858 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,858 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,858 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,858 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,858 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,858 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,858 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,858 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:27,859 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:27,859 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:27,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:27,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:27,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,861 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:27,861 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,861 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,861 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,861 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,861 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,861 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,861 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,861 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,862 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,862 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,862 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,862 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,862 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,862 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,862 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN 2024-11-09T05:40:27,862 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-09T05:40:27,862 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:27,863 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:27,863 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:27,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,865 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,865 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,865 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,865 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN to state=OFFLINE, location=null, table=test, region=b4485a6989fa449d3718b74dd5bef8f3 to restore RIT 2024-11-09T05:40:27,865 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN to state=OFFLINE, location=null, table=test, region=b609b56af9f6fc87d9a153736ec0c8d1 to restore RIT 2024-11-09T05:40:27,865 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN to state=OFFLINE, location=null, table=test, region=532f29d232f4841336f315b0406fe4af to restore RIT 2024-11-09T05:40:27,866 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN to state=OFFLINE, location=null, table=test, region=bd305d310e01738dc5cbabbfddb8eb5c to restore RIT 2024-11-09T05:40:27,866 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN to state=OFFLINE, location=null, table=test, region=8714188f3856e307485ef7426df7ea3c to restore RIT 2024-11-09T05:40:27,866 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN to state=OFFLINE, location=null, table=test, region=42ac5695b6e0e52a5e46661734af96cc to restore RIT 2024-11-09T05:40:27,866 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=348c820a04c95623395fb88f37fad187, ASSIGN to state=OFFLINE, location=null, table=test, region=348c820a04c95623395fb88f37fad187 to restore RIT 2024-11-09T05:40:27,866 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130857866 2024-11-09T05:40:27,866 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:27,866 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,868 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:27,869 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:27,869 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,869 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:27,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,876 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:27,876 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130887876 2024-11-09T05:40:27,876 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130947876 2024-11-09T05:40:27,876 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-09T05:40:27,876 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:27,876 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,876 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:27,877 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:27,877 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:27,877 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:27,877 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:27,877 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:27,878 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:27,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:27,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,880 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,881 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,881 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:27,882 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:27,882 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,883 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,884 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,884 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,884 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN 2024-11-09T05:40:27,885 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-09T05:40:27,885 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:27,886 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:27,886 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:27,886 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,889 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,889 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,889 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,890 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34809 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,890 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN to state=OFFLINE, location=null, table=test, region=b4485a6989fa449d3718b74dd5bef8f3 to restore RIT 2024-11-09T05:40:27,890 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN to state=OFFLINE, location=null, table=test, region=b609b56af9f6fc87d9a153736ec0c8d1 to restore RIT 2024-11-09T05:40:27,890 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN to state=OFFLINE, location=null, table=test, region=532f29d232f4841336f315b0406fe4af to restore RIT 2024-11-09T05:40:27,890 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN to state=OFFLINE, location=null, table=test, region=bd305d310e01738dc5cbabbfddb8eb5c to restore RIT 2024-11-09T05:40:27,890 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN to state=OFFLINE, location=null, table=test, region=8714188f3856e307485ef7426df7ea3c to restore RIT 2024-11-09T05:40:27,890 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=42ac5695b6e0e52a5e46661734af96cc, ASSIGN to state=OFFLINE, location=null, table=test, region=42ac5695b6e0e52a5e46661734af96cc to restore RIT 2024-11-09T05:40:27,891 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130857891 2024-11-09T05:40:27,891 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:27,891 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,892 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:27,893 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:27,893 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,893 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,896 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,898 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:27,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,899 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,900 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,900 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:27,900 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130887900 2024-11-09T05:40:27,900 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130947900 2024-11-09T05:40:27,900 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-09T05:40:27,900 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:27,900 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,900 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:27,900 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:27,900 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:27,900 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:27,900 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:27,901 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:27,901 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:27,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:27,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,903 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,903 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:27,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:27,904 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,905 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,905 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,905 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,905 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,905 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,906 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN 2024-11-09T05:40:27,906 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-09T05:40:27,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:27,906 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:27,907 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:27,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,908 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,909 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,909 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,909 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN to state=OFFLINE, location=null, table=test, region=b4485a6989fa449d3718b74dd5bef8f3 to restore RIT 2024-11-09T05:40:27,909 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN to state=OFFLINE, location=null, table=test, region=b609b56af9f6fc87d9a153736ec0c8d1 to restore RIT 2024-11-09T05:40:27,909 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN to state=OFFLINE, location=null, table=test, region=532f29d232f4841336f315b0406fe4af to restore RIT 2024-11-09T05:40:27,909 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN to state=OFFLINE, location=null, table=test, region=bd305d310e01738dc5cbabbfddb8eb5c to restore RIT 2024-11-09T05:40:27,909 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8714188f3856e307485ef7426df7ea3c, ASSIGN to state=OFFLINE, location=null, table=test, region=8714188f3856e307485ef7426df7ea3c to restore RIT 2024-11-09T05:40:27,910 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130857910 2024-11-09T05:40:27,910 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:27,910 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,912 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:27,913 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:27,913 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,913 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:27,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,919 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:27,919 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130887919 2024-11-09T05:40:27,919 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130947919 2024-11-09T05:40:27,919 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-09T05:40:27,919 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:27,920 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,920 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:27,920 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:27,920 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:27,920 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:27,920 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:27,920 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:27,921 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:27,923 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:27,923 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,923 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,923 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,923 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,923 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,923 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,923 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,923 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,923 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,923 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,923 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:27,924 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:27,924 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:27,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:27,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:27,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,926 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,926 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,926 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,926 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,926 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,926 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,926 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,926 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,926 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,926 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,927 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,927 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,927 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,927 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,927 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,927 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN 2024-11-09T05:40:27,927 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-09T05:40:27,928 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:27,928 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:27,928 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:27,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,930 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,930 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,930 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,931 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN to state=OFFLINE, location=null, table=test, region=b4485a6989fa449d3718b74dd5bef8f3 to restore RIT 2024-11-09T05:40:27,931 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN to state=OFFLINE, location=null, table=test, region=b609b56af9f6fc87d9a153736ec0c8d1 to restore RIT 2024-11-09T05:40:27,931 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN to state=OFFLINE, location=null, table=test, region=532f29d232f4841336f315b0406fe4af to restore RIT 2024-11-09T05:40:27,931 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bd305d310e01738dc5cbabbfddb8eb5c, ASSIGN to state=OFFLINE, location=null, table=test, region=bd305d310e01738dc5cbabbfddb8eb5c to restore RIT 2024-11-09T05:40:27,931 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130857931 2024-11-09T05:40:27,931 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:27,931 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,933 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:27,934 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:27,934 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,934 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:27,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,938 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,939 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:27,939 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130887939 2024-11-09T05:40:27,939 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130947939 2024-11-09T05:40:27,939 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-09T05:40:27,939 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:27,939 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,939 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:27,939 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:27,939 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:27,939 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:27,939 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:27,939 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:27,940 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:27,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:27,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,942 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,942 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,942 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,942 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,942 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,942 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,942 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,942 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,942 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:27,942 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:27,942 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:27,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:27,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:27,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,943 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,943 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,943 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,943 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,943 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,943 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,943 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,943 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,943 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,943 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,943 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,943 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,943 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,943 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN 2024-11-09T05:40:27,944 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-09T05:40:27,944 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:27,944 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:27,944 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:27,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,946 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,946 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,946 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,947 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN to state=OFFLINE, location=null, table=test, region=b4485a6989fa449d3718b74dd5bef8f3 to restore RIT 2024-11-09T05:40:27,947 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN to state=OFFLINE, location=null, table=test, region=b609b56af9f6fc87d9a153736ec0c8d1 to restore RIT 2024-11-09T05:40:27,947 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=532f29d232f4841336f315b0406fe4af, ASSIGN to state=OFFLINE, location=null, table=test, region=532f29d232f4841336f315b0406fe4af to restore RIT 2024-11-09T05:40:27,948 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130857948 2024-11-09T05:40:27,948 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:27,948 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,949 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:27,950 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:27,950 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,950 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,952 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,953 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:27,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,954 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,955 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,955 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,955 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,955 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,955 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,955 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,955 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,956 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,956 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,956 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:27,956 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130887956 2024-11-09T05:40:27,956 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130947956 2024-11-09T05:40:27,956 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-09T05:40:27,956 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:27,956 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,956 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:27,956 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:27,956 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:27,957 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:27,957 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:27,957 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:27,958 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:27,959 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:27,959 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,959 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,959 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,959 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,959 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,959 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,959 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,959 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:27,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:27,960 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:27,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:27,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:27,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,961 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,962 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,962 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,962 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,962 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,962 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,962 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,962 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,962 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,962 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,962 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,962 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,962 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,962 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,962 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN 2024-11-09T05:40:27,962 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-09T05:40:27,963 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:27,963 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:27,963 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:27,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,974 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,974 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,974 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,974 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN to state=OFFLINE, location=null, table=test, region=b4485a6989fa449d3718b74dd5bef8f3 to restore RIT 2024-11-09T05:40:27,975 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b609b56af9f6fc87d9a153736ec0c8d1, ASSIGN to state=OFFLINE, location=null, table=test, region=b609b56af9f6fc87d9a153736ec0c8d1 to restore RIT 2024-11-09T05:40:27,975 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130857975 2024-11-09T05:40:27,975 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:27,975 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,976 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:27,977 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:27,977 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,977 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,979 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:27,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,981 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:27,981 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130887981 2024-11-09T05:40:27,981 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130947981 2024-11-09T05:40:27,981 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-09T05:40:27,981 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:27,982 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,982 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:27,982 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:27,982 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:27,982 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:27,982 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:27,982 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:27,983 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:27,984 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:27,984 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,984 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,984 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,984 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,984 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,984 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,984 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:27,985 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:27,985 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:27,985 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:27,986 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:27,986 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,986 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,986 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:27,986 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:27,986 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,986 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,986 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,986 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,986 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,986 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,986 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,986 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,986 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,986 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:27,986 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:27,986 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN 2024-11-09T05:40:27,986 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-09T05:40:27,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:27,987 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:27,987 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:27,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:27,989 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,989 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,989 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:27,989 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b4485a6989fa449d3718b74dd5bef8f3, ASSIGN to state=OFFLINE, location=null, table=test, region=b4485a6989fa449d3718b74dd5bef8f3 to restore RIT 2024-11-09T05:40:27,990 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130857990 2024-11-09T05:40:27,990 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:27,990 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,991 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:27,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,995 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:27,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,995 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:27,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,995 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,995 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:27,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:27,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:27,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:27,997 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:27,997 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130887997 2024-11-09T05:40:27,997 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130947997 2024-11-09T05:40:27,997 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-09T05:40:27,997 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:27,997 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:27,997 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:27,997 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:27,997 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:27,997 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:27,997 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:27,998 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:27,998 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:27,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:27,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:27,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:27,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:27,999 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:28,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:28,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:28,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:28,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:28,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:28,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:28,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:28,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:28,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:28,001 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:28,001 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:28,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:28,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:28,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:28,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:28,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:28,001 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:28,001 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-09T05:40:28,002 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:28,002 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:28,002 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:28,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:28,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:28,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:28,004 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:28,004 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:28,004 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:28,005 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130858005 2024-11-09T05:40:28,005 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:28,005 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:28,007 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:28,008 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:28,008 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:28,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:28,008 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:28,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:28,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,011 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,012 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,012 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:28,012 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130888012 2024-11-09T05:40:28,012 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130948012 2024-11-09T05:40:28,012 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-09T05:40:28,013 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:28,013 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:28,013 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:28,013 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:28,013 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:28,013 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:28,013 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:28,013 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:28,014 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:28,016 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:28,016 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:28,016 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:28,016 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:28,016 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:28,016 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:28,016 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:28,017 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:28,017 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:28,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:28,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:28,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:28,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:28,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:28,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:28,018 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:28,018 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:28,018 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:28,018 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:28,018 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:28,018 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.1731130821693, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:28,018 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-09T05:40:28,019 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:28,019 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:28,019 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:28,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:28,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:28,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:28,021 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:28,021 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:28,021 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:28,022 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130858022 2024-11-09T05:40:28,022 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:28,022 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:28,023 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:28,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,028 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:28,028 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:28,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:28,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,028 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:28,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:28,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,030 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:28,030 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130888030 2024-11-09T05:40:28,030 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130948030 2024-11-09T05:40:28,030 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-09T05:40:28,031 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:28,031 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:28,031 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:28,031 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:28,031 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:28,031 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:28,031 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:28,031 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:28,032 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:28,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:28,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:28,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:28,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:28,034 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T05:40:28,034 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:28,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:28,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:28,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:28,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:28,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:28,036 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:28,036 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 2024-11-09T05:40:28,036 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:28,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:28,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:28,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:28,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e09398052c91,40411,1731130820128 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:28,036 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-09T05:40:28,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:28,037 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:28,037 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:28,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:28,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:28,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:28,040 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:28,040 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:28,040 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:28,041 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130858041 2024-11-09T05:40:28,041 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:28,041 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:28,042 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:28,043 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:28,044 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:28,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:28,044 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:28,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:28,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,049 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,049 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:28,049 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130888049 2024-11-09T05:40:28,049 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130948049 2024-11-09T05:40:28,049 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-09T05:40:28,049 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:28,049 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:28,049 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:28,049 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:28,049 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:28,049 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:28,050 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:28,050 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:28,050 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:28,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:28,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:28,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:28,052 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T05:40:28,052 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:28,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:28,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:28,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:28,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:28,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:28,054 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T05:40:28,054 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:28,054 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:28,054 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:28,054 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:28,054 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-09T05:40:28,055 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:28,055 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:28,055 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:28,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:28,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:28,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:28,057 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:28,057 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:28,057 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:28,059 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130858059 2024-11-09T05:40:28,059 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:28,059 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:28,060 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:28,062 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:28,062 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:28,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:28,062 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:28,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,064 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,065 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:28,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,066 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,067 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:28,067 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130888067 2024-11-09T05:40:28,067 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130948067 2024-11-09T05:40:28,067 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-09T05:40:28,067 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:28,068 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:28,068 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:28,068 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:28,068 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:28,068 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:28,068 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:28,068 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:28,069 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:28,071 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:28,071 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:28,071 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T05:40:28,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:28,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:28,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:28,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:28,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:28,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:28,072 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:28,072 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:28,072 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:28,072 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-09T05:40:28,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:28,073 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:28,073 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:28,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:28,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:28,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:28,075 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:28,075 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:28,075 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:28,078 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130858078 2024-11-09T05:40:28,078 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:28,078 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:28,079 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:28,080 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:28,080 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:28,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:28,080 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:28,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:28,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,084 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,084 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:28,084 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130888084 2024-11-09T05:40:28,084 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130948084 2024-11-09T05:40:28,084 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-09T05:40:28,084 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:28,084 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:28,084 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:28,084 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:28,085 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:28,085 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:28,085 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:28,085 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:28,086 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:28,087 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:28,087 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T05:40:28,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:28,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:28,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:28,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:28,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 2024-11-09T05:40:28,088 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:28,088 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure e09398052c91%2C34331%2C1731130820076.meta.1731130822074.meta, worker=e09398052c91,44035,1731130819959 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T05:40:28,088 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-09T05:40:28,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:28,088 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:28,088 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:28,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:28,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:28,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:28,090 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:28,090 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:28,090 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:28,091 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130858091 2024-11-09T05:40:28,091 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:28,091 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:28,093 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:28,094 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T05:40:28,094 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:28,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:28,094 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:28,096 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,097 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:28,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,098 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,098 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:28,098 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130888098 2024-11-09T05:40:28,098 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130948098 2024-11-09T05:40:28,098 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-09T05:40:28,099 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T05:40:28,099 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:28,099 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:28,099 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:28,099 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T05:40:28,099 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T05:40:28,099 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T05:40:28,099 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T05:40:28,100 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T05:40:28,101 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T05:40:28,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T05:40:28,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T05:40:28,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d63f643dc4acf4d7c54bcb479b29df0, REOPEN/MOVE 2024-11-09T05:40:28,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true 2024-11-09T05:40:28,102 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T05:40:28,102 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-09T05:40:28,103 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=65 2024-11-09T05:40:28,103 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=e09398052c91,40411,1731130820128, table=hbase:meta, region=1588230740 2024-11-09T05:40:28,103 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e09398052c91,40411,1731130820128, state=OPEN 2024-11-09T05:40:28,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:28,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:28,106 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:28,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T05:40:28,106 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:28,106 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T05:40:28,107 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731130858107 2024-11-09T05:40:28,107 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T05:40:28,107 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-09T05:40:28,108 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T05:40:28,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=80, ppid=72, state=RUNNABLE, hasLock=true; OpenRegionProcedure 42ac5695b6e0e52a5e46661734af96cc, server=e09398052c91,40411,1731130820128:java.lang.RuntimeException: inject code bug; ServerCrashProcedure e09398052c91,34331,1731130820076, splitWal=true, meta=true exec-time=2.2120 sec 2024-11-09T05:40:28,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=056a4a6d72fcadc66bad84500d464f5b, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fee264e99db7cad2e2fb211d7666c11, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42ac5695b6e0e52a5e46661734af96cc, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51844aa86315c53347f864258b9b5926, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b4485a6989fa449d3718b74dd5bef8f3, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=22143fefe98480e8609aa64fb5bbe23e, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df09fc3517c167e05c71f65345ca6b80, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd305d310e01738dc5cbabbfddb8eb5c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=348c820a04c95623395fb88f37fad187, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6ea3613715cd8a321124b73881db15ce, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=706d9970ba371fdec8f2a079940ca3ae, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12126c11ff675bc823b55beab9656286, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6c8538a5c08a601164d6d90f17409b33, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ce006577ade7d81e0bf87827254771f5, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d63f643dc4acf4d7c54bcb479b29df0, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=5 2024-11-09T05:40:28,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d014bc791a8473500fbbfa044123f955, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d8a7929b729c2e22434e4a1d52e461a, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8714188f3856e307485ef7426df7ea3c, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c06857dd8e828d035329855bf73ed41f, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2339918a4a54d7d8274a43da565fe2a2, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=532f29d232f4841336f315b0406fe4af, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f1e5e5fd2628827f76a773887689bc8, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cc6d417b0bdc1061e6457e90a27badd, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d923c8e1af5892d9575aa4b3336f28f9, regionState=OPEN, lastHost=e09398052c91,44035,1731130819959, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b609b56af9f6fc87d9a153736ec0c8d1, regionState=OPENING, lastHost=e09398052c91,34331,1731130820076, regionLocation=e09398052c91,44035,1731130819959, openSeqNum=2 2024-11-09T05:40:28,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6874567d13af4b499b6eb873decadd10, regionState=OPEN, lastHost=e09398052c91,40411,1731130820128, regionLocation=e09398052c91,40411,1731130820128, openSeqNum=2 2024-11-09T05:40:28,115 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-09T05:40:28,115 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731130888115 2024-11-09T05:40:28,116 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731130948115 2024-11-09T05:40:28,116 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-09T05:40:28,140 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=350 (was 378), OpenFileDescriptor=604 (was 623), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=296 (was 296), ProcessCount=11 (was 11), AvailableMemoryMB=3435 (was 3496) 2024-11-09T05:40:28,141 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-09T05:40:28,141 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T05:40:28,141 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T05:40:28,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T05:40:28,142 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T05:40:28,142 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T05:40:28,142 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-09T05:40:28,142 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1372924334, stopped=false 2024-11-09T05:40:28,143 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e09398052c91,34809,1731130819107 2024-11-09T05:40:28,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T05:40:28,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T05:40:28,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T05:40:28,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:28,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:28,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:28,145 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:28,145 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T05:40:28,145 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T05:40:28,145 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T05:40:28,145 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T05:40:28,146 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T05:40:28,146 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e09398052c91,44035,1731130819959' ***** 2024-11-09T05:40:28,146 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-09T05:40:28,146 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T05:40:28,146 INFO [RS:0;e09398052c91:44035 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T05:40:28,146 INFO [RS:0;e09398052c91:44035 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T05:40:28,146 INFO [RS:0;e09398052c91:44035 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T05:40:28,147 INFO [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(3091): Received CLOSE for b609b56af9f6fc87d9a153736ec0c8d1 2024-11-09T05:40:28,147 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T05:40:28,147 INFO [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(3091): Received CLOSE for 8714188f3856e307485ef7426df7ea3c 2024-11-09T05:40:28,147 INFO [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(3091): Received CLOSE for 6c8538a5c08a601164d6d90f17409b33 2024-11-09T05:40:28,147 INFO [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(3091): Received CLOSE for b4485a6989fa449d3718b74dd5bef8f3 2024-11-09T05:40:28,147 INFO [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(3091): Received CLOSE for d014bc791a8473500fbbfa044123f955 2024-11-09T05:40:28,147 INFO [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(3091): Received CLOSE for 6ea3613715cd8a321124b73881db15ce 2024-11-09T05:40:28,147 INFO [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(3091): Received CLOSE for 9fee264e99db7cad2e2fb211d7666c11 2024-11-09T05:40:28,147 INFO [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(3091): Received CLOSE for c06857dd8e828d035329855bf73ed41f 2024-11-09T05:40:28,147 INFO [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(3091): Received CLOSE for 22143fefe98480e8609aa64fb5bbe23e 2024-11-09T05:40:28,147 INFO [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(3091): Received CLOSE for d923c8e1af5892d9575aa4b3336f28f9 2024-11-09T05:40:28,147 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e09398052c91,40411,1731130820128' ***** 2024-11-09T05:40:28,147 INFO [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(3091): Received CLOSE for 0f1e5e5fd2628827f76a773887689bc8 2024-11-09T05:40:28,148 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-09T05:40:28,148 INFO [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(3091): Received CLOSE for 348c820a04c95623395fb88f37fad187 2024-11-09T05:40:28,148 INFO [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(3091): Received CLOSE for 8d63f643dc4acf4d7c54bcb479b29df0 2024-11-09T05:40:28,148 INFO [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(959): stopping server e09398052c91,44035,1731130819959 2024-11-09T05:40:28,148 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b609b56af9f6fc87d9a153736ec0c8d1, disabling compactions & flushes 2024-11-09T05:40:28,148 INFO [RS:0;e09398052c91:44035 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T05:40:28,148 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1. 2024-11-09T05:40:28,148 INFO [RS:2;e09398052c91:40411 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T05:40:28,148 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1. 2024-11-09T05:40:28,148 INFO [RS:0;e09398052c91:44035 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e09398052c91:44035. 2024-11-09T05:40:28,148 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1. after waiting 0 ms 2024-11-09T05:40:28,148 INFO [RS:2;e09398052c91:40411 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T05:40:28,148 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T05:40:28,148 DEBUG [RS:0;e09398052c91:44035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T05:40:28,148 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1. 2024-11-09T05:40:28,148 INFO [RS:2;e09398052c91:40411 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T05:40:28,148 DEBUG [RS:0;e09398052c91:44035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T05:40:28,148 INFO [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(3091): Received CLOSE for 51844aa86315c53347f864258b9b5926 2024-11-09T05:40:28,148 INFO [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-09T05:40:28,149 DEBUG [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(1351): Waiting on 0f1e5e5fd2628827f76a773887689bc8, 22143fefe98480e8609aa64fb5bbe23e, 348c820a04c95623395fb88f37fad187, 6c8538a5c08a601164d6d90f17409b33, 6ea3613715cd8a321124b73881db15ce, 8714188f3856e307485ef7426df7ea3c, 8d63f643dc4acf4d7c54bcb479b29df0, 9fee264e99db7cad2e2fb211d7666c11, b4485a6989fa449d3718b74dd5bef8f3, b609b56af9f6fc87d9a153736ec0c8d1, c06857dd8e828d035329855bf73ed41f, d014bc791a8473500fbbfa044123f955, d923c8e1af5892d9575aa4b3336f28f9 2024-11-09T05:40:28,149 INFO [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(3091): Received CLOSE for 532f29d232f4841336f315b0406fe4af 2024-11-09T05:40:28,149 INFO [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(3091): Received CLOSE for 2339918a4a54d7d8274a43da565fe2a2 2024-11-09T05:40:28,149 INFO [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(3091): Received CLOSE for 12126c11ff675bc823b55beab9656286 2024-11-09T05:40:28,149 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 51844aa86315c53347f864258b9b5926, disabling compactions & flushes 2024-11-09T05:40:28,149 INFO [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(3091): Received CLOSE for 056a4a6d72fcadc66bad84500d464f5b 2024-11-09T05:40:28,150 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1731130822466.51844aa86315c53347f864258b9b5926. 2024-11-09T05:40:28,150 INFO [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(3091): Received CLOSE for 1cc6d417b0bdc1061e6457e90a27badd 2024-11-09T05:40:28,150 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731130822466.51844aa86315c53347f864258b9b5926. 2024-11-09T05:40:28,150 INFO [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(3091): Received CLOSE for 8d8a7929b729c2e22434e4a1d52e461a 2024-11-09T05:40:28,150 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731130822466.51844aa86315c53347f864258b9b5926. after waiting 0 ms 2024-11-09T05:40:28,150 INFO [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(3091): Received CLOSE for 706d9970ba371fdec8f2a079940ca3ae 2024-11-09T05:40:28,150 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731130822466.51844aa86315c53347f864258b9b5926. 2024-11-09T05:40:28,150 INFO [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(3091): Received CLOSE for 6874567d13af4b499b6eb873decadd10 2024-11-09T05:40:28,150 INFO [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(3091): Received CLOSE for 42ac5695b6e0e52a5e46661734af96cc 2024-11-09T05:40:28,150 INFO [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(3091): Received CLOSE for df09fc3517c167e05c71f65345ca6b80 2024-11-09T05:40:28,150 INFO [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(3091): Received CLOSE for ce006577ade7d81e0bf87827254771f5 2024-11-09T05:40:28,150 INFO [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(3091): Received CLOSE for bd305d310e01738dc5cbabbfddb8eb5c 2024-11-09T05:40:28,150 INFO [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(959): stopping server e09398052c91,40411,1731130820128 2024-11-09T05:40:28,150 INFO [RS:2;e09398052c91:40411 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T05:40:28,150 INFO [RS:2;e09398052c91:40411 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;e09398052c91:40411. 2024-11-09T05:40:28,150 DEBUG [RS:2;e09398052c91:40411 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T05:40:28,150 DEBUG [RS:2;e09398052c91:40411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T05:40:28,151 INFO [RS:2;e09398052c91:40411 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T05:40:28,151 INFO [RS:2;e09398052c91:40411 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T05:40:28,151 INFO [RS:2;e09398052c91:40411 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T05:40:28,151 INFO [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-09T05:40:28,151 INFO [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-09T05:40:28,151 DEBUG [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(1351): Waiting on 056a4a6d72fcadc66bad84500d464f5b, 12126c11ff675bc823b55beab9656286, 1588230740, 1cc6d417b0bdc1061e6457e90a27badd, 2339918a4a54d7d8274a43da565fe2a2, 42ac5695b6e0e52a5e46661734af96cc, 51844aa86315c53347f864258b9b5926, 532f29d232f4841336f315b0406fe4af, 6874567d13af4b499b6eb873decadd10, 706d9970ba371fdec8f2a079940ca3ae, 8d8a7929b729c2e22434e4a1d52e461a, bd305d310e01738dc5cbabbfddb8eb5c, ce006577ade7d81e0bf87827254771f5, df09fc3517c167e05c71f65345ca6b80 2024-11-09T05:40:28,151 DEBUG [RS_CLOSE_META-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T05:40:28,151 INFO [RS_CLOSE_META-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T05:40:28,151 DEBUG [RS_CLOSE_META-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T05:40:28,151 DEBUG [RS_CLOSE_META-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T05:40:28,151 DEBUG [RS_CLOSE_META-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T05:40:28,152 INFO [RS_CLOSE_META-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-11-09T05:40:28,155 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/b609b56af9f6fc87d9a153736ec0c8d1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,156 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1. 2024-11-09T05:40:28,156 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b609b56af9f6fc87d9a153736ec0c8d1: Waiting for close lock at 1731130828147Running coprocessor pre-close hooks at 1731130828148 (+1 ms)Disabling compacts and flushes for region at 1731130828148Disabling writes for close at 1731130828148Writing region close event to WAL at 1731130828149 (+1 ms)Running coprocessor post-close hooks at 1731130828156 (+7 ms)Closed at 1731130828156 2024-11-09T05:40:28,156 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1731130822466.b609b56af9f6fc87d9a153736ec0c8d1. 2024-11-09T05:40:28,156 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8714188f3856e307485ef7426df7ea3c, disabling compactions & flushes 2024-11-09T05:40:28,156 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/51844aa86315c53347f864258b9b5926/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,156 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c. 2024-11-09T05:40:28,156 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c. 2024-11-09T05:40:28,156 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c. after waiting 0 ms 2024-11-09T05:40:28,156 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c. 2024-11-09T05:40:28,157 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1731130822466.51844aa86315c53347f864258b9b5926. 2024-11-09T05:40:28,157 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 51844aa86315c53347f864258b9b5926: Waiting for close lock at 1731130828149Running coprocessor pre-close hooks at 1731130828149Disabling compacts and flushes for region at 1731130828149Disabling writes for close at 1731130828150 (+1 ms)Writing region close event to WAL at 1731130828151 (+1 ms)Running coprocessor post-close hooks at 1731130828157 (+6 ms)Closed at 1731130828157 2024-11-09T05:40:28,157 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1731130822466.51844aa86315c53347f864258b9b5926. 2024-11-09T05:40:28,157 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 532f29d232f4841336f315b0406fe4af, disabling compactions & flushes 2024-11-09T05:40:28,157 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af. 2024-11-09T05:40:28,157 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af. 2024-11-09T05:40:28,157 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af. after waiting 0 ms 2024-11-09T05:40:28,157 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af. 2024-11-09T05:40:28,162 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/8714188f3856e307485ef7426df7ea3c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,162 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c. 2024-11-09T05:40:28,163 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8714188f3856e307485ef7426df7ea3c: Waiting for close lock at 1731130828156Running coprocessor pre-close hooks at 1731130828156Disabling compacts and flushes for region at 1731130828156Disabling writes for close at 1731130828156Writing region close event to WAL at 1731130828157 (+1 ms)Running coprocessor post-close hooks at 1731130828162 (+5 ms)Closed at 1731130828162 2024-11-09T05:40:28,163 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1731130822466.8714188f3856e307485ef7426df7ea3c. 2024-11-09T05:40:28,163 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/532f29d232f4841336f315b0406fe4af/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,163 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6c8538a5c08a601164d6d90f17409b33, disabling compactions & flushes 2024-11-09T05:40:28,163 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1731130822466.6c8538a5c08a601164d6d90f17409b33. 2024-11-09T05:40:28,163 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731130822466.6c8538a5c08a601164d6d90f17409b33. 2024-11-09T05:40:28,163 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731130822466.6c8538a5c08a601164d6d90f17409b33. after waiting 0 ms 2024-11-09T05:40:28,163 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731130822466.6c8538a5c08a601164d6d90f17409b33. 2024-11-09T05:40:28,164 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af. 2024-11-09T05:40:28,164 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 532f29d232f4841336f315b0406fe4af: Waiting for close lock at 1731130828157Running coprocessor pre-close hooks at 1731130828157Disabling compacts and flushes for region at 1731130828157Disabling writes for close at 1731130828157Writing region close event to WAL at 1731130828158 (+1 ms)Running coprocessor post-close hooks at 1731130828163 (+5 ms)Closed at 1731130828163 2024-11-09T05:40:28,164 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1731130822466.532f29d232f4841336f315b0406fe4af. 2024-11-09T05:40:28,164 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2339918a4a54d7d8274a43da565fe2a2, disabling compactions & flushes 2024-11-09T05:40:28,164 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1731130822466.2339918a4a54d7d8274a43da565fe2a2. 2024-11-09T05:40:28,164 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731130822466.2339918a4a54d7d8274a43da565fe2a2. 2024-11-09T05:40:28,164 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731130822466.2339918a4a54d7d8274a43da565fe2a2. after waiting 0 ms 2024-11-09T05:40:28,164 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731130822466.2339918a4a54d7d8274a43da565fe2a2. 2024-11-09T05:40:28,167 DEBUG [RS_CLOSE_META-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/.tmp/info/d2e8b4cb7801421dafc6b9c4521a94f3 is 138, key is test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc./info:regioninfo/1731130827205/Put/seqid=0 2024-11-09T05:40:28,168 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/6c8538a5c08a601164d6d90f17409b33/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,169 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1731130822466.6c8538a5c08a601164d6d90f17409b33. 2024-11-09T05:40:28,169 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6c8538a5c08a601164d6d90f17409b33: Waiting for close lock at 1731130828163Running coprocessor pre-close hooks at 1731130828163Disabling compacts and flushes for region at 1731130828163Disabling writes for close at 1731130828163Writing region close event to WAL at 1731130828163Running coprocessor post-close hooks at 1731130828168 (+5 ms)Closed at 1731130828169 (+1 ms) 2024-11-09T05:40:28,169 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1731130822466.6c8538a5c08a601164d6d90f17409b33. 2024-11-09T05:40:28,169 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b4485a6989fa449d3718b74dd5bef8f3, disabling compactions & flushes 2024-11-09T05:40:28,169 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3. 2024-11-09T05:40:28,169 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3. 2024-11-09T05:40:28,169 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3. after waiting 0 ms 2024-11-09T05:40:28,169 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3. 2024-11-09T05:40:28,176 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/2339918a4a54d7d8274a43da565fe2a2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,177 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1731130822466.2339918a4a54d7d8274a43da565fe2a2. 2024-11-09T05:40:28,177 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2339918a4a54d7d8274a43da565fe2a2: Waiting for close lock at 1731130828164Running coprocessor pre-close hooks at 1731130828164Disabling compacts and flushes for region at 1731130828164Disabling writes for close at 1731130828164Writing region close event to WAL at 1731130828165 (+1 ms)Running coprocessor post-close hooks at 1731130828177 (+12 ms)Closed at 1731130828177 2024-11-09T05:40:28,177 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1731130822466.2339918a4a54d7d8274a43da565fe2a2. 2024-11-09T05:40:28,178 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 12126c11ff675bc823b55beab9656286, disabling compactions & flushes 2024-11-09T05:40:28,178 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1731130822466.12126c11ff675bc823b55beab9656286. 2024-11-09T05:40:28,178 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731130822466.12126c11ff675bc823b55beab9656286. 2024-11-09T05:40:28,178 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731130822466.12126c11ff675bc823b55beab9656286. after waiting 0 ms 2024-11-09T05:40:28,178 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731130822466.12126c11ff675bc823b55beab9656286. 2024-11-09T05:40:28,179 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/b4485a6989fa449d3718b74dd5bef8f3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,180 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3. 2024-11-09T05:40:28,180 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b4485a6989fa449d3718b74dd5bef8f3: Waiting for close lock at 1731130828169Running coprocessor pre-close hooks at 1731130828169Disabling compacts and flushes for region at 1731130828169Disabling writes for close at 1731130828169Writing region close event to WAL at 1731130828175 (+6 ms)Running coprocessor post-close hooks at 1731130828180 (+5 ms)Closed at 1731130828180 2024-11-09T05:40:28,180 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1731130822466.b4485a6989fa449d3718b74dd5bef8f3. 2024-11-09T05:40:28,180 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d014bc791a8473500fbbfa044123f955, disabling compactions & flushes 2024-11-09T05:40:28,180 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1731130822466.d014bc791a8473500fbbfa044123f955. 2024-11-09T05:40:28,181 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731130822466.d014bc791a8473500fbbfa044123f955. 2024-11-09T05:40:28,181 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731130822466.d014bc791a8473500fbbfa044123f955. after waiting 0 ms 2024-11-09T05:40:28,181 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731130822466.d014bc791a8473500fbbfa044123f955. 2024-11-09T05:40:28,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741869_1045 (size=8195) 2024-11-09T05:40:28,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741869_1045 (size=8195) 2024-11-09T05:40:28,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741869_1045 (size=8195) 2024-11-09T05:40:28,184 INFO [RS_CLOSE_META-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/.tmp/info/d2e8b4cb7801421dafc6b9c4521a94f3 2024-11-09T05:40:28,185 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/12126c11ff675bc823b55beab9656286/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,186 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1731130822466.12126c11ff675bc823b55beab9656286. 2024-11-09T05:40:28,186 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 12126c11ff675bc823b55beab9656286: Waiting for close lock at 1731130828178Running coprocessor pre-close hooks at 1731130828178Disabling compacts and flushes for region at 1731130828178Disabling writes for close at 1731130828178Writing region close event to WAL at 1731130828179 (+1 ms)Running coprocessor post-close hooks at 1731130828186 (+7 ms)Closed at 1731130828186 2024-11-09T05:40:28,186 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1731130822466.12126c11ff675bc823b55beab9656286. 2024-11-09T05:40:28,187 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 056a4a6d72fcadc66bad84500d464f5b, disabling compactions & flushes 2024-11-09T05:40:28,187 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1731130822466.056a4a6d72fcadc66bad84500d464f5b. 2024-11-09T05:40:28,187 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731130822466.056a4a6d72fcadc66bad84500d464f5b. 2024-11-09T05:40:28,187 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1731130822466.056a4a6d72fcadc66bad84500d464f5b. after waiting 0 ms 2024-11-09T05:40:28,187 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1731130822466.056a4a6d72fcadc66bad84500d464f5b. 2024-11-09T05:40:28,187 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/d014bc791a8473500fbbfa044123f955/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,188 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1731130822466.d014bc791a8473500fbbfa044123f955. 2024-11-09T05:40:28,188 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d014bc791a8473500fbbfa044123f955: Waiting for close lock at 1731130828180Running coprocessor pre-close hooks at 1731130828180Disabling compacts and flushes for region at 1731130828180Disabling writes for close at 1731130828181 (+1 ms)Writing region close event to WAL at 1731130828183 (+2 ms)Running coprocessor post-close hooks at 1731130828188 (+5 ms)Closed at 1731130828188 2024-11-09T05:40:28,188 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1731130822466.d014bc791a8473500fbbfa044123f955. 2024-11-09T05:40:28,188 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6ea3613715cd8a321124b73881db15ce, disabling compactions & flushes 2024-11-09T05:40:28,188 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1731130822466.6ea3613715cd8a321124b73881db15ce. 2024-11-09T05:40:28,188 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731130822466.6ea3613715cd8a321124b73881db15ce. 2024-11-09T05:40:28,188 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731130822466.6ea3613715cd8a321124b73881db15ce. after waiting 0 ms 2024-11-09T05:40:28,188 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731130822466.6ea3613715cd8a321124b73881db15ce. 2024-11-09T05:40:28,194 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/6ea3613715cd8a321124b73881db15ce/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,194 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/056a4a6d72fcadc66bad84500d464f5b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,195 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1731130822466.6ea3613715cd8a321124b73881db15ce. 2024-11-09T05:40:28,195 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6ea3613715cd8a321124b73881db15ce: Waiting for close lock at 1731130828188Running coprocessor pre-close hooks at 1731130828188Disabling compacts and flushes for region at 1731130828188Disabling writes for close at 1731130828188Writing region close event to WAL at 1731130828189 (+1 ms)Running coprocessor post-close hooks at 1731130828195 (+6 ms)Closed at 1731130828195 2024-11-09T05:40:28,195 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1731130822466.6ea3613715cd8a321124b73881db15ce. 2024-11-09T05:40:28,195 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9fee264e99db7cad2e2fb211d7666c11, disabling compactions & flushes 2024-11-09T05:40:28,195 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1731130822466.056a4a6d72fcadc66bad84500d464f5b. 2024-11-09T05:40:28,195 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1731130822466.9fee264e99db7cad2e2fb211d7666c11. 2024-11-09T05:40:28,195 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 056a4a6d72fcadc66bad84500d464f5b: Waiting for close lock at 1731130828186Running coprocessor pre-close hooks at 1731130828186Disabling compacts and flushes for region at 1731130828187 (+1 ms)Disabling writes for close at 1731130828187Writing region close event to WAL at 1731130828187Running coprocessor post-close hooks at 1731130828195 (+8 ms)Closed at 1731130828195 2024-11-09T05:40:28,195 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731130822466.9fee264e99db7cad2e2fb211d7666c11. 2024-11-09T05:40:28,195 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731130822466.9fee264e99db7cad2e2fb211d7666c11. after waiting 0 ms 2024-11-09T05:40:28,195 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731130822466.9fee264e99db7cad2e2fb211d7666c11. 2024-11-09T05:40:28,195 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1731130822466.056a4a6d72fcadc66bad84500d464f5b. 2024-11-09T05:40:28,196 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1cc6d417b0bdc1061e6457e90a27badd, disabling compactions & flushes 2024-11-09T05:40:28,196 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1731130822466.1cc6d417b0bdc1061e6457e90a27badd. 2024-11-09T05:40:28,196 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731130822466.1cc6d417b0bdc1061e6457e90a27badd. 2024-11-09T05:40:28,196 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1731130822466.1cc6d417b0bdc1061e6457e90a27badd. after waiting 0 ms 2024-11-09T05:40:28,196 DEBUG [RS_CLOSE_META-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/.tmp/info/d2e8b4cb7801421dafc6b9c4521a94f3 as hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/info/d2e8b4cb7801421dafc6b9c4521a94f3 2024-11-09T05:40:28,196 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1731130822466.1cc6d417b0bdc1061e6457e90a27badd. 2024-11-09T05:40:28,200 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/1cc6d417b0bdc1061e6457e90a27badd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,201 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1731130822466.1cc6d417b0bdc1061e6457e90a27badd. 2024-11-09T05:40:28,201 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1cc6d417b0bdc1061e6457e90a27badd: Waiting for close lock at 1731130828196Running coprocessor pre-close hooks at 1731130828196Disabling compacts and flushes for region at 1731130828196Disabling writes for close at 1731130828196Writing region close event to WAL at 1731130828197 (+1 ms)Running coprocessor post-close hooks at 1731130828201 (+4 ms)Closed at 1731130828201 2024-11-09T05:40:28,202 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/9fee264e99db7cad2e2fb211d7666c11/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,202 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1731130822466.1cc6d417b0bdc1061e6457e90a27badd. 2024-11-09T05:40:28,202 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8d8a7929b729c2e22434e4a1d52e461a, disabling compactions & flushes 2024-11-09T05:40:28,202 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1731130822466.8d8a7929b729c2e22434e4a1d52e461a. 2024-11-09T05:40:28,202 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731130822466.8d8a7929b729c2e22434e4a1d52e461a. 2024-11-09T05:40:28,202 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731130822466.8d8a7929b729c2e22434e4a1d52e461a. after waiting 0 ms 2024-11-09T05:40:28,202 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731130822466.8d8a7929b729c2e22434e4a1d52e461a. 2024-11-09T05:40:28,202 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1731130822466.9fee264e99db7cad2e2fb211d7666c11. 2024-11-09T05:40:28,203 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9fee264e99db7cad2e2fb211d7666c11: Waiting for close lock at 1731130828195Running coprocessor pre-close hooks at 1731130828195Disabling compacts and flushes for region at 1731130828195Disabling writes for close at 1731130828195Writing region close event to WAL at 1731130828196 (+1 ms)Running coprocessor post-close hooks at 1731130828202 (+6 ms)Closed at 1731130828202 2024-11-09T05:40:28,203 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1731130822466.9fee264e99db7cad2e2fb211d7666c11. 2024-11-09T05:40:28,203 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c06857dd8e828d035329855bf73ed41f, disabling compactions & flushes 2024-11-09T05:40:28,203 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1731130822466.c06857dd8e828d035329855bf73ed41f. 2024-11-09T05:40:28,203 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731130822466.c06857dd8e828d035329855bf73ed41f. 2024-11-09T05:40:28,203 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731130822466.c06857dd8e828d035329855bf73ed41f. after waiting 0 ms 2024-11-09T05:40:28,203 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731130822466.c06857dd8e828d035329855bf73ed41f. 2024-11-09T05:40:28,205 INFO [RS_CLOSE_META-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/info/d2e8b4cb7801421dafc6b9c4521a94f3, entries=24, sequenceid=75, filesize=8.0 K 2024-11-09T05:40:28,207 INFO [RS_CLOSE_META-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 56ms, sequenceid=75, compaction requested=false 2024-11-09T05:40:28,208 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/c06857dd8e828d035329855bf73ed41f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,209 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1731130822466.c06857dd8e828d035329855bf73ed41f. 2024-11-09T05:40:28,209 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c06857dd8e828d035329855bf73ed41f: Waiting for close lock at 1731130828203Running coprocessor pre-close hooks at 1731130828203Disabling compacts and flushes for region at 1731130828203Disabling writes for close at 1731130828203Writing region close event to WAL at 1731130828204 (+1 ms)Running coprocessor post-close hooks at 1731130828209 (+5 ms)Closed at 1731130828209 2024-11-09T05:40:28,209 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1731130822466.c06857dd8e828d035329855bf73ed41f. 2024-11-09T05:40:28,209 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 22143fefe98480e8609aa64fb5bbe23e, disabling compactions & flushes 2024-11-09T05:40:28,209 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1731130822466.22143fefe98480e8609aa64fb5bbe23e. 2024-11-09T05:40:28,209 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731130822466.22143fefe98480e8609aa64fb5bbe23e. 2024-11-09T05:40:28,209 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731130822466.22143fefe98480e8609aa64fb5bbe23e. after waiting 0 ms 2024-11-09T05:40:28,209 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731130822466.22143fefe98480e8609aa64fb5bbe23e. 2024-11-09T05:40:28,213 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/8d8a7929b729c2e22434e4a1d52e461a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,214 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1731130822466.8d8a7929b729c2e22434e4a1d52e461a. 2024-11-09T05:40:28,214 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8d8a7929b729c2e22434e4a1d52e461a: Waiting for close lock at 1731130828202Running coprocessor pre-close hooks at 1731130828202Disabling compacts and flushes for region at 1731130828202Disabling writes for close at 1731130828202Writing region close event to WAL at 1731130828203 (+1 ms)Running coprocessor post-close hooks at 1731130828214 (+11 ms)Closed at 1731130828214 2024-11-09T05:40:28,214 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1731130822466.8d8a7929b729c2e22434e4a1d52e461a. 2024-11-09T05:40:28,214 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 706d9970ba371fdec8f2a079940ca3ae, disabling compactions & flushes 2024-11-09T05:40:28,214 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1731130822466.706d9970ba371fdec8f2a079940ca3ae. 2024-11-09T05:40:28,214 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731130822466.706d9970ba371fdec8f2a079940ca3ae. 2024-11-09T05:40:28,214 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731130822466.706d9970ba371fdec8f2a079940ca3ae. after waiting 0 ms 2024-11-09T05:40:28,214 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731130822466.706d9970ba371fdec8f2a079940ca3ae. 2024-11-09T05:40:28,217 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/22143fefe98480e8609aa64fb5bbe23e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,217 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1731130822466.22143fefe98480e8609aa64fb5bbe23e. 2024-11-09T05:40:28,218 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 22143fefe98480e8609aa64fb5bbe23e: Waiting for close lock at 1731130828209Running coprocessor pre-close hooks at 1731130828209Disabling compacts and flushes for region at 1731130828209Disabling writes for close at 1731130828209Writing region close event to WAL at 1731130828213 (+4 ms)Running coprocessor post-close hooks at 1731130828217 (+4 ms)Closed at 1731130828217 2024-11-09T05:40:28,218 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1731130822466.22143fefe98480e8609aa64fb5bbe23e. 2024-11-09T05:40:28,218 DEBUG [RS_CLOSE_META-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-11-09T05:40:28,218 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d923c8e1af5892d9575aa4b3336f28f9, disabling compactions & flushes 2024-11-09T05:40:28,218 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1731130822466.d923c8e1af5892d9575aa4b3336f28f9. 2024-11-09T05:40:28,218 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731130822466.d923c8e1af5892d9575aa4b3336f28f9. 2024-11-09T05:40:28,218 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731130822466.d923c8e1af5892d9575aa4b3336f28f9. after waiting 0 ms 2024-11-09T05:40:28,218 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731130822466.d923c8e1af5892d9575aa4b3336f28f9. 2024-11-09T05:40:28,219 DEBUG [RS_CLOSE_META-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T05:40:28,219 INFO [RS_CLOSE_META-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T05:40:28,219 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/706d9970ba371fdec8f2a079940ca3ae/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,219 DEBUG [RS_CLOSE_META-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731130828151Running coprocessor pre-close hooks at 1731130828151Disabling compacts and flushes for region at 1731130828151Disabling writes for close at 1731130828151Obtaining lock to block concurrent updates at 1731130828152 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731130828152Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1731130828159 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731130828160 (+1 ms)Flushing 1588230740/info: creating writer at 1731130828160Flushing 1588230740/info: appending metadata at 1731130828166 (+6 ms)Flushing 1588230740/info: closing flushed file at 1731130828166Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@220083db: reopening flushed file at 1731130828194 (+28 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 56ms, sequenceid=75, compaction requested=false at 1731130828207 (+13 ms)Writing region close event to WAL at 1731130828213 (+6 ms)Running coprocessor post-close hooks at 1731130828219 (+6 ms)Closed at 1731130828219 2024-11-09T05:40:28,220 DEBUG [RS_CLOSE_META-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-09T05:40:28,220 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1731130822466.706d9970ba371fdec8f2a079940ca3ae. 2024-11-09T05:40:28,220 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 706d9970ba371fdec8f2a079940ca3ae: Waiting for close lock at 1731130828214Running coprocessor pre-close hooks at 1731130828214Disabling compacts and flushes for region at 1731130828214Disabling writes for close at 1731130828214Writing region close event to WAL at 1731130828215 (+1 ms)Running coprocessor post-close hooks at 1731130828220 (+5 ms)Closed at 1731130828220 2024-11-09T05:40:28,220 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1731130822466.706d9970ba371fdec8f2a079940ca3ae. 2024-11-09T05:40:28,221 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6874567d13af4b499b6eb873decadd10, disabling compactions & flushes 2024-11-09T05:40:28,221 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1731130822466.6874567d13af4b499b6eb873decadd10. 2024-11-09T05:40:28,221 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731130822466.6874567d13af4b499b6eb873decadd10. 2024-11-09T05:40:28,221 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731130822466.6874567d13af4b499b6eb873decadd10. after waiting 0 ms 2024-11-09T05:40:28,221 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731130822466.6874567d13af4b499b6eb873decadd10. 2024-11-09T05:40:28,222 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/d923c8e1af5892d9575aa4b3336f28f9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,223 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1731130822466.d923c8e1af5892d9575aa4b3336f28f9. 2024-11-09T05:40:28,223 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d923c8e1af5892d9575aa4b3336f28f9: Waiting for close lock at 1731130828218Running coprocessor pre-close hooks at 1731130828218Disabling compacts and flushes for region at 1731130828218Disabling writes for close at 1731130828218Writing region close event to WAL at 1731130828218Running coprocessor post-close hooks at 1731130828223 (+5 ms)Closed at 1731130828223 2024-11-09T05:40:28,223 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1731130822466.d923c8e1af5892d9575aa4b3336f28f9. 2024-11-09T05:40:28,223 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0f1e5e5fd2628827f76a773887689bc8, disabling compactions & flushes 2024-11-09T05:40:28,223 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1731130822466.0f1e5e5fd2628827f76a773887689bc8. 2024-11-09T05:40:28,224 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731130822466.0f1e5e5fd2628827f76a773887689bc8. 2024-11-09T05:40:28,224 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731130822466.0f1e5e5fd2628827f76a773887689bc8. after waiting 0 ms 2024-11-09T05:40:28,224 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731130822466.0f1e5e5fd2628827f76a773887689bc8. 2024-11-09T05:40:28,228 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/0f1e5e5fd2628827f76a773887689bc8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,229 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1731130822466.0f1e5e5fd2628827f76a773887689bc8. 2024-11-09T05:40:28,229 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/6874567d13af4b499b6eb873decadd10/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,229 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0f1e5e5fd2628827f76a773887689bc8: Waiting for close lock at 1731130828223Running coprocessor pre-close hooks at 1731130828223Disabling compacts and flushes for region at 1731130828223Disabling writes for close at 1731130828224 (+1 ms)Writing region close event to WAL at 1731130828224Running coprocessor post-close hooks at 1731130828229 (+5 ms)Closed at 1731130828229 2024-11-09T05:40:28,229 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1731130822466.0f1e5e5fd2628827f76a773887689bc8. 2024-11-09T05:40:28,229 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 348c820a04c95623395fb88f37fad187, disabling compactions & flushes 2024-11-09T05:40:28,229 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1731130822466.348c820a04c95623395fb88f37fad187. 2024-11-09T05:40:28,229 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731130822466.348c820a04c95623395fb88f37fad187. 2024-11-09T05:40:28,229 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731130822466.348c820a04c95623395fb88f37fad187. after waiting 0 ms 2024-11-09T05:40:28,229 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731130822466.348c820a04c95623395fb88f37fad187. 2024-11-09T05:40:28,229 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1731130822466.6874567d13af4b499b6eb873decadd10. 2024-11-09T05:40:28,229 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6874567d13af4b499b6eb873decadd10: Waiting for close lock at 1731130828221Running coprocessor pre-close hooks at 1731130828221Disabling compacts and flushes for region at 1731130828221Disabling writes for close at 1731130828221Writing region close event to WAL at 1731130828221Running coprocessor post-close hooks at 1731130828229 (+8 ms)Closed at 1731130828229 2024-11-09T05:40:28,230 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1731130822466.6874567d13af4b499b6eb873decadd10. 2024-11-09T05:40:28,230 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 42ac5695b6e0e52a5e46661734af96cc, disabling compactions & flushes 2024-11-09T05:40:28,230 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc. 2024-11-09T05:40:28,230 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc. 2024-11-09T05:40:28,230 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc. after waiting 0 ms 2024-11-09T05:40:28,230 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc. 2024-11-09T05:40:28,235 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/348c820a04c95623395fb88f37fad187/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,235 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/42ac5695b6e0e52a5e46661734af96cc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,235 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1731130822466.348c820a04c95623395fb88f37fad187. 2024-11-09T05:40:28,235 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc. 2024-11-09T05:40:28,235 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 348c820a04c95623395fb88f37fad187: Waiting for close lock at 1731130828229Running coprocessor pre-close hooks at 1731130828229Disabling compacts and flushes for region at 1731130828229Disabling writes for close at 1731130828229Writing region close event to WAL at 1731130828230 (+1 ms)Running coprocessor post-close hooks at 1731130828235 (+5 ms)Closed at 1731130828235 2024-11-09T05:40:28,236 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 42ac5695b6e0e52a5e46661734af96cc: Waiting for close lock at 1731130828230Running coprocessor pre-close hooks at 1731130828230Disabling compacts and flushes for region at 1731130828230Disabling writes for close at 1731130828230Writing region close event to WAL at 1731130828230Running coprocessor post-close hooks at 1731130828235 (+5 ms)Closed at 1731130828235 2024-11-09T05:40:28,236 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1731130822466.348c820a04c95623395fb88f37fad187. 2024-11-09T05:40:28,236 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1731130822466.42ac5695b6e0e52a5e46661734af96cc. 2024-11-09T05:40:28,236 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8d63f643dc4acf4d7c54bcb479b29df0, disabling compactions & flushes 2024-11-09T05:40:28,236 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0. 2024-11-09T05:40:28,236 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing df09fc3517c167e05c71f65345ca6b80, disabling compactions & flushes 2024-11-09T05:40:28,236 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0. 2024-11-09T05:40:28,236 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1731130822466.df09fc3517c167e05c71f65345ca6b80. 2024-11-09T05:40:28,236 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0. after waiting 0 ms 2024-11-09T05:40:28,236 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731130822466.df09fc3517c167e05c71f65345ca6b80. 2024-11-09T05:40:28,236 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0. 2024-11-09T05:40:28,236 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731130822466.df09fc3517c167e05c71f65345ca6b80. after waiting 0 ms 2024-11-09T05:40:28,236 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731130822466.df09fc3517c167e05c71f65345ca6b80. 2024-11-09T05:40:28,241 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/df09fc3517c167e05c71f65345ca6b80/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,242 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/8d63f643dc4acf4d7c54bcb479b29df0/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-09T05:40:28,242 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1731130822466.df09fc3517c167e05c71f65345ca6b80. 2024-11-09T05:40:28,242 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for df09fc3517c167e05c71f65345ca6b80: Waiting for close lock at 1731130828236Running coprocessor pre-close hooks at 1731130828236Disabling compacts and flushes for region at 1731130828236Disabling writes for close at 1731130828236Writing region close event to WAL at 1731130828236Running coprocessor post-close hooks at 1731130828242 (+6 ms)Closed at 1731130828242 2024-11-09T05:40:28,242 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0. 2024-11-09T05:40:28,243 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8d63f643dc4acf4d7c54bcb479b29df0: Waiting for close lock at 1731130828236Running coprocessor pre-close hooks at 1731130828236Disabling compacts and flushes for region at 1731130828236Disabling writes for close at 1731130828236Writing region close event to WAL at 1731130828236Running coprocessor post-close hooks at 1731130828242 (+6 ms)Closed at 1731130828242 2024-11-09T05:40:28,243 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1731130822466.df09fc3517c167e05c71f65345ca6b80. 2024-11-09T05:40:28,243 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1731130822466.8d63f643dc4acf4d7c54bcb479b29df0. 2024-11-09T05:40:28,243 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ce006577ade7d81e0bf87827254771f5, disabling compactions & flushes 2024-11-09T05:40:28,243 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1731130822466.ce006577ade7d81e0bf87827254771f5. 2024-11-09T05:40:28,243 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731130822466.ce006577ade7d81e0bf87827254771f5. 2024-11-09T05:40:28,243 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731130822466.ce006577ade7d81e0bf87827254771f5. after waiting 0 ms 2024-11-09T05:40:28,243 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731130822466.ce006577ade7d81e0bf87827254771f5. 2024-11-09T05:40:28,244 INFO [regionserver/e09398052c91:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T05:40:28,244 INFO [regionserver/e09398052c91:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T05:40:28,247 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/ce006577ade7d81e0bf87827254771f5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,248 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1731130822466.ce006577ade7d81e0bf87827254771f5. 2024-11-09T05:40:28,248 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ce006577ade7d81e0bf87827254771f5: Waiting for close lock at 1731130828243Running coprocessor pre-close hooks at 1731130828243Disabling compacts and flushes for region at 1731130828243Disabling writes for close at 1731130828243Writing region close event to WAL at 1731130828243Running coprocessor post-close hooks at 1731130828247 (+4 ms)Closed at 1731130828248 (+1 ms) 2024-11-09T05:40:28,248 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1731130822466.ce006577ade7d81e0bf87827254771f5. 2024-11-09T05:40:28,248 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bd305d310e01738dc5cbabbfddb8eb5c, disabling compactions & flushes 2024-11-09T05:40:28,248 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c. 2024-11-09T05:40:28,248 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c. 2024-11-09T05:40:28,248 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c. after waiting 0 ms 2024-11-09T05:40:28,248 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c. 2024-11-09T05:40:28,252 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/data/default/test/bd305d310e01738dc5cbabbfddb8eb5c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T05:40:28,253 INFO [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c. 2024-11-09T05:40:28,253 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bd305d310e01738dc5cbabbfddb8eb5c: Waiting for close lock at 1731130828248Running coprocessor pre-close hooks at 1731130828248Disabling compacts and flushes for region at 1731130828248Disabling writes for close at 1731130828248Writing region close event to WAL at 1731130828248Running coprocessor post-close hooks at 1731130828253 (+5 ms)Closed at 1731130828253 2024-11-09T05:40:28,253 DEBUG [RS_CLOSE_REGION-regionserver/e09398052c91:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1731130822466.bd305d310e01738dc5cbabbfddb8eb5c. 2024-11-09T05:40:28,349 INFO [RS:0;e09398052c91:44035 {}] regionserver.HRegionServer(976): stopping server e09398052c91,44035,1731130819959; all regions closed. 2024-11-09T05:40:28,351 INFO [RS:2;e09398052c91:40411 {}] regionserver.HRegionServer(976): stopping server e09398052c91,40411,1731130820128; all regions closed. 2024-11-09T05:40:28,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741834_1010 (size=7074) 2024-11-09T05:40:28,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741834_1010 (size=7074) 2024-11-09T05:40:28,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741834_1010 (size=7074) 2024-11-09T05:40:28,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741865_1041 (size=4204) 2024-11-09T05:40:28,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741865_1041 (size=4204) 2024-11-09T05:40:28,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741865_1041 (size=4204) 2024-11-09T05:40:28,357 DEBUG [RS:0;e09398052c91:44035 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/oldWALs 2024-11-09T05:40:28,358 INFO [RS:0;e09398052c91:44035 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e09398052c91%2C44035%2C1731130819959:(num 1731130821693) 2024-11-09T05:40:28,358 DEBUG [RS:0;e09398052c91:44035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T05:40:28,358 INFO [RS:0;e09398052c91:44035 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T05:40:28,358 INFO [RS:0;e09398052c91:44035 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T05:40:28,358 INFO [RS:0;e09398052c91:44035 {}] hbase.ChoreService(370): Chore service for: regionserver/e09398052c91:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-09T05:40:28,358 INFO [RS:0;e09398052c91:44035 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T05:40:28,359 INFO [RS:0;e09398052c91:44035 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T05:40:28,359 INFO [regionserver/e09398052c91:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T05:40:28,359 INFO [RS:0;e09398052c91:44035 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T05:40:28,359 INFO [RS:0;e09398052c91:44035 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T05:40:28,359 DEBUG [RS:2;e09398052c91:40411 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/oldWALs 2024-11-09T05:40:28,359 INFO [RS:2;e09398052c91:40411 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e09398052c91%2C40411%2C1731130820128.meta:.meta(num 1731130826517) 2024-11-09T05:40:28,359 INFO [RS:0;e09398052c91:44035 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44035 2024-11-09T05:40:28,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x100fb3de9450001, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e09398052c91,44035,1731130819959 2024-11-09T05:40:28,361 INFO [RS:0;e09398052c91:44035 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T05:40:28,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T05:40:28,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741835_1011 (size=7064) 2024-11-09T05:40:28,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741835_1011 (size=7064) 2024-11-09T05:40:28,363 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e09398052c91,44035,1731130819959] 2024-11-09T05:40:28,365 DEBUG [RS:2;e09398052c91:40411 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/oldWALs 2024-11-09T05:40:28,365 INFO [RS:2;e09398052c91:40411 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e09398052c91%2C40411%2C1731130820128:(num 1731130821693) 2024-11-09T05:40:28,365 DEBUG [RS:2;e09398052c91:40411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T05:40:28,365 INFO [RS:2;e09398052c91:40411 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T05:40:28,365 INFO [RS:2;e09398052c91:40411 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T05:40:28,365 INFO [RS:2;e09398052c91:40411 {}] hbase.ChoreService(370): Chore service for: regionserver/e09398052c91:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-09T05:40:28,366 INFO [RS:2;e09398052c91:40411 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T05:40:28,366 INFO [regionserver/e09398052c91:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T05:40:28,366 INFO [RS:2;e09398052c91:40411 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40411 2024-11-09T05:40:28,366 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e09398052c91,44035,1731130819959 already deleted, retry=false 2024-11-09T05:40:28,367 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e09398052c91,44035,1731130819959 expired; onlineServers=1 2024-11-09T05:40:28,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T05:40:28,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40411-0x100fb3de9450003, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e09398052c91,40411,1731130820128 2024-11-09T05:40:28,368 INFO [RS:2;e09398052c91:40411 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T05:40:28,369 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e09398052c91,40411,1731130820128] 2024-11-09T05:40:28,370 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e09398052c91,40411,1731130820128 already deleted, retry=false 2024-11-09T05:40:28,371 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e09398052c91,40411,1731130820128 expired; onlineServers=0 2024-11-09T05:40:28,371 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e09398052c91,34809,1731130819107' ***** 2024-11-09T05:40:28,371 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-09T05:40:28,371 INFO [M:0;e09398052c91:34809 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T05:40:28,371 INFO [M:0;e09398052c91:34809 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T05:40:28,371 DEBUG [M:0;e09398052c91:34809 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-09T05:40:28,371 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-09T05:40:28,371 DEBUG [M:0;e09398052c91:34809 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-09T05:40:28,371 DEBUG [master/e09398052c91:0:becomeActiveMaster-HFileCleaner.large.0-1731130821330 {}] cleaner.HFileCleaner(306): Exit Thread[master/e09398052c91:0:becomeActiveMaster-HFileCleaner.large.0-1731130821330,5,FailOnTimeoutGroup] 2024-11-09T05:40:28,371 DEBUG [master/e09398052c91:0:becomeActiveMaster-HFileCleaner.small.0-1731130821332 {}] cleaner.HFileCleaner(306): Exit Thread[master/e09398052c91:0:becomeActiveMaster-HFileCleaner.small.0-1731130821332,5,FailOnTimeoutGroup] 2024-11-09T05:40:28,371 INFO [M:0;e09398052c91:34809 {}] hbase.ChoreService(370): Chore service for: master/e09398052c91:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-09T05:40:28,371 INFO [M:0;e09398052c91:34809 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T05:40:28,372 DEBUG [M:0;e09398052c91:34809 {}] master.HMaster(1795): Stopping service threads 2024-11-09T05:40:28,372 INFO [M:0;e09398052c91:34809 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-09T05:40:28,372 INFO [M:0;e09398052c91:34809 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T05:40:28,372 INFO [M:0;e09398052c91:34809 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T05:40:28,372 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-09T05:40:28,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-09T05:40:28,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T05:40:28,374 DEBUG [M:0;e09398052c91:34809 {}] zookeeper.ZKUtil(347): master:34809-0x100fb3de9450000, quorum=127.0.0.1:51298, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-09T05:40:28,374 WARN [M:0;e09398052c91:34809 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-09T05:40:28,375 INFO [M:0;e09398052c91:34809 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/.lastflushedseqids 2024-11-09T05:40:28,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741870_1046 (size=1) 2024-11-09T05:40:28,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741870_1046 (size=1) 2024-11-09T05:40:28,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741870_1046 (size=1) 2024-11-09T05:40:28,389 INFO [M:0;e09398052c91:34809 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T05:40:28,390 INFO [M:0;e09398052c91:34809 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-09T05:40:28,390 DEBUG [M:0;e09398052c91:34809 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T05:40:28,390 INFO [M:0;e09398052c91:34809 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T05:40:28,390 DEBUG [M:0;e09398052c91:34809 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T05:40:28,390 DEBUG [M:0;e09398052c91:34809 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T05:40:28,390 DEBUG [M:0;e09398052c91:34809 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T05:40:28,390 INFO [M:0;e09398052c91:34809 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=260.86 KB heapSize=317.06 KB 2024-11-09T05:40:28,410 DEBUG [M:0;e09398052c91:34809 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c9411cdbf5a740d9ad0c6818f2dc8142 is 82, key is hbase:meta,,1/info:regioninfo/1731130826827/Put/seqid=0 2024-11-09T05:40:28,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39285 is added to blk_1073741871_1047 (size=6071) 2024-11-09T05:40:28,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34201 is added to blk_1073741871_1047 (size=6071) 2024-11-09T05:40:28,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43869 is added to blk_1073741871_1047 (size=6071) 2024-11-09T05:40:28,423 INFO [M:0;e09398052c91:34809 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=603 (bloomFilter=true), to=hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c9411cdbf5a740d9ad0c6818f2dc8142 2024-11-09T05:40:28,449 DEBUG [M:0;e09398052c91:34809 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43215/user/jenkins/test-data/6c109b4d-5c83-4c8e-5725-196c01896352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/86dc911cf1434fe98923e0f5a109ba58 is 2990, key is \x00\x00\x00\x00\x00\x00\x00