2024-12-05 13:17:05,758 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-12-05 13:17:05,792 main DEBUG Took 0.032338 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-05 13:17:05,793 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-05 13:17:05,793 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-05 13:17:05,795 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-05 13:17:05,797 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:17:05,804 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-05 13:17:05,818 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:17:05,819 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:17:05,820 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:17:05,821 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:17:05,821 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:17:05,822 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:17:05,823 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:17:05,823 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:17:05,824 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:17:05,824 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:17:05,825 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:17:05,826 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:17:05,827 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:17:05,827 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:17:05,828 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:17:05,828 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:17:05,829 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:17:05,829 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:17:05,830 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:17:05,830 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:17:05,831 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:17:05,831 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:17:05,832 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:17:05,832 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 13:17:05,833 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:17:05,833 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-05 13:17:05,835 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 13:17:05,837 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-05 13:17:05,841 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-05 13:17:05,842 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-05 13:17:05,843 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-05 13:17:05,844 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-05 13:17:05,858 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-05 13:17:05,861 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-05 13:17:05,863 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-05 13:17:05,863 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-05 13:17:05,864 main DEBUG createAppenders(={Console}) 2024-12-05 13:17:05,865 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-12-05 13:17:05,865 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-12-05 13:17:05,866 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-12-05 13:17:05,866 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-05 13:17:05,867 main DEBUG OutputStream closed 2024-12-05 13:17:05,867 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-05 13:17:05,867 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-05 13:17:05,868 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-12-05 13:17:05,949 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-05 13:17:05,953 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-05 13:17:05,954 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-05 13:17:05,956 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-05 13:17:05,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-05 13:17:05,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-05 13:17:05,958 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-05 13:17:05,958 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-05 13:17:05,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-05 13:17:05,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-05 13:17:05,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-05 13:17:05,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-05 13:17:05,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-05 13:17:05,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-05 13:17:05,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-05 13:17:05,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-05 13:17:05,962 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-05 13:17:05,963 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-05 13:17:05,966 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05 13:17:05,967 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-12-05 13:17:05,967 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-05 13:17:05,968 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-12-05T13:17:05,993 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-12-05 13:17:05,996 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-05 13:17:05,997 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05T13:17:06,306 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8 2024-12-05T13:17:06,349 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-05T13:17:06,371 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/cluster_75ee48d7-b897-e19d-61ba-662deded8c0f, deleteOnExit=true 2024-12-05T13:17:06,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-05T13:17:06,373 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/test.cache.data in system properties and HBase conf 2024-12-05T13:17:06,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/hadoop.tmp.dir in system properties and HBase conf 2024-12-05T13:17:06,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/hadoop.log.dir in system properties and HBase conf 2024-12-05T13:17:06,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-05T13:17:06,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-05T13:17:06,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-05T13:17:06,497 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-05T13:17:06,675 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T13:17:06,685 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-05T13:17:06,686 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-05T13:17:06,687 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-05T13:17:06,688 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T13:17:06,689 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-05T13:17:06,689 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-05T13:17:06,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T13:17:06,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T13:17:06,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-05T13:17:06,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/nfs.dump.dir in system properties and HBase conf 2024-12-05T13:17:06,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/java.io.tmpdir in system properties and HBase conf 2024-12-05T13:17:06,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T13:17:06,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-05T13:17:06,693 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-05T13:17:07,909 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-05T13:17:07,999 INFO [Time-limited test {}] log.Log(170): Logging initialized @2891ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-05T13:17:08,094 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T13:17:08,189 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T13:17:08,237 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T13:17:08,238 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T13:17:08,240 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T13:17:08,260 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T13:17:08,264 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/hadoop.log.dir/,AVAILABLE} 2024-12-05T13:17:08,265 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T13:17:08,495 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/java.io.tmpdir/jetty-localhost-44053-hadoop-hdfs-3_4_1-tests_jar-_-any-2917482188419659133/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T13:17:08,505 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:44053} 2024-12-05T13:17:08,506 INFO [Time-limited test {}] server.Server(415): Started @3398ms 2024-12-05T13:17:09,243 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T13:17:09,253 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T13:17:09,255 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T13:17:09,256 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T13:17:09,256 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T13:17:09,264 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@51c39823{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/hadoop.log.dir/,AVAILABLE} 2024-12-05T13:17:09,265 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c7135d9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T13:17:09,405 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@65e17246{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/java.io.tmpdir/jetty-localhost-38167-hadoop-hdfs-3_4_1-tests_jar-_-any-2200112093104044308/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T13:17:09,410 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@220c8d80{HTTP/1.1, (http/1.1)}{localhost:38167} 2024-12-05T13:17:09,410 INFO [Time-limited test {}] server.Server(415): Started @4303ms 2024-12-05T13:17:09,486 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T13:17:09,681 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T13:17:09,688 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T13:17:09,694 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T13:17:09,695 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T13:17:09,695 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T13:17:09,699 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2ff36c4c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/hadoop.log.dir/,AVAILABLE} 2024-12-05T13:17:09,701 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6331ed3f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T13:17:09,845 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@395ff33a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/java.io.tmpdir/jetty-localhost-45377-hadoop-hdfs-3_4_1-tests_jar-_-any-12373763122060692515/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T13:17:09,847 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4d24ac65{HTTP/1.1, (http/1.1)}{localhost:45377} 2024-12-05T13:17:09,848 INFO [Time-limited test {}] server.Server(415): Started @4740ms 2024-12-05T13:17:09,852 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T13:17:09,953 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T13:17:09,959 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T13:17:09,970 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T13:17:09,971 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T13:17:09,971 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T13:17:09,974 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6ed43cd1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/hadoop.log.dir/,AVAILABLE} 2024-12-05T13:17:09,975 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48a2d1c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T13:17:10,093 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@46041a9d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/java.io.tmpdir/jetty-localhost-40341-hadoop-hdfs-3_4_1-tests_jar-_-any-15671520028563615258/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T13:17:10,094 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2a39ea25{HTTP/1.1, (http/1.1)}{localhost:40341} 2024-12-05T13:17:10,095 INFO [Time-limited test {}] server.Server(415): Started @4987ms 2024-12-05T13:17:10,097 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T13:17:10,858 WARN [Thread-121 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/cluster_75ee48d7-b897-e19d-61ba-662deded8c0f/data/data2/current/BP-1141445696-172.17.0.3-1733404627457/current, will proceed with Du for space computation calculation, 2024-12-05T13:17:10,858 WARN [Thread-120 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/cluster_75ee48d7-b897-e19d-61ba-662deded8c0f/data/data1/current/BP-1141445696-172.17.0.3-1733404627457/current, will proceed with Du for space computation calculation, 2024-12-05T13:17:10,899 WARN [Thread-128 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/cluster_75ee48d7-b897-e19d-61ba-662deded8c0f/data/data3/current/BP-1141445696-172.17.0.3-1733404627457/current, will proceed with Du for space computation calculation, 2024-12-05T13:17:10,899 WARN [Thread-129 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/cluster_75ee48d7-b897-e19d-61ba-662deded8c0f/data/data4/current/BP-1141445696-172.17.0.3-1733404627457/current, will proceed with Du for space computation calculation, 2024-12-05T13:17:10,905 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T13:17:10,938 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T13:17:10,965 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb9d66364f023e072 with lease ID 0xc029324ebdeb9564: Processing first storage report for DS-0c343192-22cb-4379-b3b1-2cf2bdfac1b3 from datanode DatanodeRegistration(127.0.0.1:42229, datanodeUuid=ab30316d-c6dc-4bb6-a173-454b37469f43, infoPort=33987, infoSecurePort=0, ipcPort=44629, storageInfo=lv=-57;cid=testClusterID;nsid=37629280;c=1733404627457) 2024-12-05T13:17:10,966 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb9d66364f023e072 with lease ID 0xc029324ebdeb9564: from storage DS-0c343192-22cb-4379-b3b1-2cf2bdfac1b3 node DatanodeRegistration(127.0.0.1:42229, datanodeUuid=ab30316d-c6dc-4bb6-a173-454b37469f43, infoPort=33987, infoSecurePort=0, ipcPort=44629, storageInfo=lv=-57;cid=testClusterID;nsid=37629280;c=1733404627457), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-05T13:17:10,967 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x42c849b62706d53 with lease ID 0xc029324ebdeb9565: Processing first storage report for DS-d2ff1f2a-33de-4af6-9d9e-1d533285e26a from datanode DatanodeRegistration(127.0.0.1:34109, datanodeUuid=2d5b7502-29e4-4247-91f9-ca32ae6a5af6, infoPort=40917, infoSecurePort=0, ipcPort=42341, storageInfo=lv=-57;cid=testClusterID;nsid=37629280;c=1733404627457) 2024-12-05T13:17:10,967 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x42c849b62706d53 with lease ID 0xc029324ebdeb9565: from storage DS-d2ff1f2a-33de-4af6-9d9e-1d533285e26a node DatanodeRegistration(127.0.0.1:34109, datanodeUuid=2d5b7502-29e4-4247-91f9-ca32ae6a5af6, infoPort=40917, infoSecurePort=0, ipcPort=42341, storageInfo=lv=-57;cid=testClusterID;nsid=37629280;c=1733404627457), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T13:17:10,967 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb9d66364f023e072 with lease ID 0xc029324ebdeb9564: Processing first storage report for DS-5dce427a-7ec0-4bec-bb90-59cc7b5ee1d0 from datanode DatanodeRegistration(127.0.0.1:42229, datanodeUuid=ab30316d-c6dc-4bb6-a173-454b37469f43, infoPort=33987, infoSecurePort=0, ipcPort=44629, storageInfo=lv=-57;cid=testClusterID;nsid=37629280;c=1733404627457) 2024-12-05T13:17:10,967 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb9d66364f023e072 with lease ID 0xc029324ebdeb9564: from storage DS-5dce427a-7ec0-4bec-bb90-59cc7b5ee1d0 node DatanodeRegistration(127.0.0.1:42229, datanodeUuid=ab30316d-c6dc-4bb6-a173-454b37469f43, infoPort=33987, infoSecurePort=0, ipcPort=44629, storageInfo=lv=-57;cid=testClusterID;nsid=37629280;c=1733404627457), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-05T13:17:10,968 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x42c849b62706d53 with lease ID 0xc029324ebdeb9565: Processing first storage report for DS-d2b51639-ceaf-43bf-b1c0-853344295890 from datanode DatanodeRegistration(127.0.0.1:34109, datanodeUuid=2d5b7502-29e4-4247-91f9-ca32ae6a5af6, infoPort=40917, infoSecurePort=0, ipcPort=42341, storageInfo=lv=-57;cid=testClusterID;nsid=37629280;c=1733404627457) 2024-12-05T13:17:10,968 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x42c849b62706d53 with lease ID 0xc029324ebdeb9565: from storage DS-d2b51639-ceaf-43bf-b1c0-853344295890 node DatanodeRegistration(127.0.0.1:34109, datanodeUuid=2d5b7502-29e4-4247-91f9-ca32ae6a5af6, infoPort=40917, infoSecurePort=0, ipcPort=42341, storageInfo=lv=-57;cid=testClusterID;nsid=37629280;c=1733404627457), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T13:17:11,107 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/cluster_75ee48d7-b897-e19d-61ba-662deded8c0f/data/data5/current/BP-1141445696-172.17.0.3-1733404627457/current, will proceed with Du for space computation calculation, 2024-12-05T13:17:11,107 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/cluster_75ee48d7-b897-e19d-61ba-662deded8c0f/data/data6/current/BP-1141445696-172.17.0.3-1733404627457/current, will proceed with Du for space computation calculation, 2024-12-05T13:17:11,141 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T13:17:11,147 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe3246e5189d48b7b with lease ID 0xc029324ebdeb9566: Processing first storage report for DS-da32b628-72dd-49f3-b291-1fa8797500ab from datanode DatanodeRegistration(127.0.0.1:32831, datanodeUuid=8f66d2e2-73bf-4e91-9c27-85f4257afcf9, infoPort=34519, infoSecurePort=0, ipcPort=39379, storageInfo=lv=-57;cid=testClusterID;nsid=37629280;c=1733404627457) 2024-12-05T13:17:11,148 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe3246e5189d48b7b with lease ID 0xc029324ebdeb9566: from storage DS-da32b628-72dd-49f3-b291-1fa8797500ab node DatanodeRegistration(127.0.0.1:32831, datanodeUuid=8f66d2e2-73bf-4e91-9c27-85f4257afcf9, infoPort=34519, infoSecurePort=0, ipcPort=39379, storageInfo=lv=-57;cid=testClusterID;nsid=37629280;c=1733404627457), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-05T13:17:11,148 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe3246e5189d48b7b with lease ID 0xc029324ebdeb9566: Processing first storage report for DS-5b75b20d-64df-4438-94a9-bc3414718844 from datanode DatanodeRegistration(127.0.0.1:32831, datanodeUuid=8f66d2e2-73bf-4e91-9c27-85f4257afcf9, infoPort=34519, infoSecurePort=0, ipcPort=39379, storageInfo=lv=-57;cid=testClusterID;nsid=37629280;c=1733404627457) 2024-12-05T13:17:11,148 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe3246e5189d48b7b with lease ID 0xc029324ebdeb9566: from storage DS-5b75b20d-64df-4438-94a9-bc3414718844 node DatanodeRegistration(127.0.0.1:32831, datanodeUuid=8f66d2e2-73bf-4e91-9c27-85f4257afcf9, infoPort=34519, infoSecurePort=0, ipcPort=39379, storageInfo=lv=-57;cid=testClusterID;nsid=37629280;c=1733404627457), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T13:17:11,167 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8 2024-12-05T13:17:11,255 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/cluster_75ee48d7-b897-e19d-61ba-662deded8c0f/zookeeper_0, clientPort=50894, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/cluster_75ee48d7-b897-e19d-61ba-662deded8c0f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/cluster_75ee48d7-b897-e19d-61ba-662deded8c0f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T13:17:11,268 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50894 2024-12-05T13:17:11,281 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T13:17:11,284 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T13:17:11,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741825_1001 (size=7) 2024-12-05T13:17:11,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741825_1001 (size=7) 2024-12-05T13:17:11,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741825_1001 (size=7) 2024-12-05T13:17:11,633 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 with version=8 2024-12-05T13:17:11,634 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/hbase-staging 2024-12-05T13:17:11,755 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-05T13:17:12,035 INFO [Time-limited test {}] client.ConnectionUtils(128): master/88aaa70743e7:0 server-side Connection retries=45 2024-12-05T13:17:12,046 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T13:17:12,047 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T13:17:12,055 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T13:17:12,055 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T13:17:12,056 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T13:17:12,220 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-05T13:17:12,279 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-05T13:17:12,287 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-05T13:17:12,292 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T13:17:12,325 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 12845 (auto-detected) 2024-12-05T13:17:12,326 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-05T13:17:12,353 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42815 2024-12-05T13:17:12,388 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42815 connecting to ZooKeeper ensemble=127.0.0.1:50894 2024-12-05T13:17:12,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:428150x0, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T13:17:12,503 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42815-0x101a6d1666f0000 connected 2024-12-05T13:17:12,589 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T13:17:12,593 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T13:17:12,603 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T13:17:12,607 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75, hbase.cluster.distributed=false 2024-12-05T13:17:12,630 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T13:17:12,635 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42815 2024-12-05T13:17:12,635 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42815 2024-12-05T13:17:12,636 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42815 2024-12-05T13:17:12,636 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42815 2024-12-05T13:17:12,637 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42815 2024-12-05T13:17:12,738 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/88aaa70743e7:0 server-side Connection retries=45 2024-12-05T13:17:12,740 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T13:17:12,740 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T13:17:12,740 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T13:17:12,740 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T13:17:12,740 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T13:17:12,743 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T13:17:12,745 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T13:17:12,746 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39293 2024-12-05T13:17:12,748 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39293 connecting to ZooKeeper ensemble=127.0.0.1:50894 2024-12-05T13:17:12,750 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T13:17:12,752 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T13:17:12,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:392930x0, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T13:17:12,768 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39293-0x101a6d1666f0001 connected 2024-12-05T13:17:12,768 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T13:17:12,773 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T13:17:12,781 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T13:17:12,783 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T13:17:12,788 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T13:17:12,789 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39293 2024-12-05T13:17:12,790 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39293 2024-12-05T13:17:12,790 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39293 2024-12-05T13:17:12,791 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39293 2024-12-05T13:17:12,791 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39293 2024-12-05T13:17:12,807 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/88aaa70743e7:0 server-side Connection retries=45 2024-12-05T13:17:12,807 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T13:17:12,808 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T13:17:12,808 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T13:17:12,808 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T13:17:12,808 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T13:17:12,808 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T13:17:12,809 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T13:17:12,810 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36055 2024-12-05T13:17:12,811 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36055 connecting to ZooKeeper ensemble=127.0.0.1:50894 2024-12-05T13:17:12,812 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T13:17:12,815 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T13:17:12,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:360550x0, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T13:17:12,826 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:360550x0, quorum=127.0.0.1:50894, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T13:17:12,826 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36055-0x101a6d1666f0002 connected 2024-12-05T13:17:12,826 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T13:17:12,827 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T13:17:12,828 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T13:17:12,830 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T13:17:12,831 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36055 2024-12-05T13:17:12,831 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36055 2024-12-05T13:17:12,832 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36055 2024-12-05T13:17:12,833 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36055 2024-12-05T13:17:12,833 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36055 2024-12-05T13:17:12,848 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/88aaa70743e7:0 server-side Connection retries=45 2024-12-05T13:17:12,848 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T13:17:12,848 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T13:17:12,849 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T13:17:12,849 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T13:17:12,849 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T13:17:12,849 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T13:17:12,849 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T13:17:12,850 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41495 2024-12-05T13:17:12,852 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41495 connecting to ZooKeeper ensemble=127.0.0.1:50894 2024-12-05T13:17:12,853 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T13:17:12,855 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T13:17:12,867 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:414950x0, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T13:17:12,868 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:414950x0, quorum=127.0.0.1:50894, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T13:17:12,868 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41495-0x101a6d1666f0003 connected 2024-12-05T13:17:12,868 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T13:17:12,870 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T13:17:12,872 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41495-0x101a6d1666f0003, quorum=127.0.0.1:50894, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T13:17:12,874 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41495-0x101a6d1666f0003, quorum=127.0.0.1:50894, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T13:17:12,876 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41495 2024-12-05T13:17:12,877 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41495 2024-12-05T13:17:12,879 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41495 2024-12-05T13:17:12,879 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41495 2024-12-05T13:17:12,880 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41495 2024-12-05T13:17:12,895 DEBUG [M:0;88aaa70743e7:42815 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;88aaa70743e7:42815 2024-12-05T13:17:12,897 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/88aaa70743e7,42815,1733404631821 2024-12-05T13:17:12,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T13:17:12,909 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41495-0x101a6d1666f0003, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T13:17:12,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T13:17:12,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T13:17:12,911 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/88aaa70743e7,42815,1733404631821 2024-12-05T13:17:12,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T13:17:12,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:12,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T13:17:12,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:12,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:12,938 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41495-0x101a6d1666f0003, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T13:17:12,938 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41495-0x101a6d1666f0003, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:12,938 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T13:17:12,940 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/88aaa70743e7,42815,1733404631821 from backup master directory 2024-12-05T13:17:12,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/88aaa70743e7,42815,1733404631821 2024-12-05T13:17:12,950 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41495-0x101a6d1666f0003, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T13:17:12,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T13:17:12,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T13:17:12,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T13:17:12,951 WARN [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T13:17:12,952 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=88aaa70743e7,42815,1733404631821 2024-12-05T13:17:12,954 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-05T13:17:12,957 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-05T13:17:13,016 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/hbase.id] with ID: d611f681-e8be-43c0-b7da-5bfaf5772f57 2024-12-05T13:17:13,017 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/.tmp/hbase.id 2024-12-05T13:17:13,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741826_1002 (size=42) 2024-12-05T13:17:13,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741826_1002 (size=42) 2024-12-05T13:17:13,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741826_1002 (size=42) 2024-12-05T13:17:13,036 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/.tmp/hbase.id]:[hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/hbase.id] 2024-12-05T13:17:13,089 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T13:17:13,096 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-05T13:17:13,116 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-12-05T13:17:13,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:13,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:13,129 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41495-0x101a6d1666f0003, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:13,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:13,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741827_1003 (size=196) 2024-12-05T13:17:13,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741827_1003 (size=196) 2024-12-05T13:17:13,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741827_1003 (size=196) 2024-12-05T13:17:13,180 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T13:17:13,183 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T13:17:13,208 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:17:13,214 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:17:13,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741828_1004 (size=1189) 2024-12-05T13:17:13,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741828_1004 (size=1189) 2024-12-05T13:17:13,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741828_1004 (size=1189) 2024-12-05T13:17:13,278 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store 2024-12-05T13:17:13,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741829_1005 (size=34) 2024-12-05T13:17:13,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741829_1005 (size=34) 2024-12-05T13:17:13,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741829_1005 (size=34) 2024-12-05T13:17:13,313 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-05T13:17:13,317 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:13,319 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T13:17:13,319 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T13:17:13,319 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T13:17:13,321 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T13:17:13,322 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T13:17:13,322 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T13:17:13,324 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733404633319Disabling compacts and flushes for region at 1733404633319Disabling writes for close at 1733404633322 (+3 ms)Writing region close event to WAL at 1733404633322Closed at 1733404633322 2024-12-05T13:17:13,327 WARN [master/88aaa70743e7:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/.initializing 2024-12-05T13:17:13,327 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/WALs/88aaa70743e7,42815,1733404631821 2024-12-05T13:17:13,335 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T13:17:13,353 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=88aaa70743e7%2C42815%2C1733404631821, suffix=, logDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/WALs/88aaa70743e7,42815,1733404631821, archiveDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/oldWALs, maxLogs=10 2024-12-05T13:17:13,388 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/WALs/88aaa70743e7,42815,1733404631821/88aaa70743e7%2C42815%2C1733404631821.1733404633360, exclude list is [], retry=0 2024-12-05T13:17:13,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42229,DS-0c343192-22cb-4379-b3b1-2cf2bdfac1b3,DISK] 2024-12-05T13:17:13,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32831,DS-da32b628-72dd-49f3-b291-1fa8797500ab,DISK] 2024-12-05T13:17:13,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34109,DS-d2ff1f2a-33de-4af6-9d9e-1d533285e26a,DISK] 2024-12-05T13:17:13,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-05T13:17:13,464 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/WALs/88aaa70743e7,42815,1733404631821/88aaa70743e7%2C42815%2C1733404631821.1733404633360 2024-12-05T13:17:13,465 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33987:33987),(127.0.0.1/127.0.0.1:34519:34519),(127.0.0.1/127.0.0.1:40917:40917)] 2024-12-05T13:17:13,466 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:17:13,467 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:13,471 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:17:13,473 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:17:13,514 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:17:13,539 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T13:17:13,543 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:13,547 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:17:13,547 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:17:13,551 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T13:17:13,551 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:13,553 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:13,553 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:17:13,556 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T13:17:13,556 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:13,558 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:13,558 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:17:13,561 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T13:17:13,561 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:13,562 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:13,563 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:17:13,566 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:17:13,567 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:17:13,573 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:17:13,573 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:17:13,577 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T13:17:13,582 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T13:17:13,587 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:13,588 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68008500, jitterRate=0.01340562105178833}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T13:17:13,594 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733404633488Initializing all the Stores at 1733404633491 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733404633491Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404633492 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404633493 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404633493Cleaning up temporary data from old regions at 1733404633573 (+80 ms)Region opened successfully at 1733404633594 (+21 ms) 2024-12-05T13:17:13,596 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T13:17:13,629 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@443cdf77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=88aaa70743e7/172.17.0.3:0 2024-12-05T13:17:13,660 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-05T13:17:13,676 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:13,677 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:13,681 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:13,683 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-05T13:17:13,690 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-05T13:17:13,691 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T13:17:13,723 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T13:17:13,732 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T13:17:13,750 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-05T13:17:13,753 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T13:17:13,755 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T13:17:13,766 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-05T13:17:13,768 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T13:17:13,772 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T13:17:13,783 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-05T13:17:13,785 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T13:17:13,800 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T13:17:13,820 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T13:17:13,829 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T13:17:13,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T13:17:13,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T13:17:13,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T13:17:13,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:13,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:13,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:13,841 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41495-0x101a6d1666f0003, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T13:17:13,842 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41495-0x101a6d1666f0003, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:13,845 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=88aaa70743e7,42815,1733404631821, sessionid=0x101a6d1666f0000, setting cluster-up flag (Was=false) 2024-12-05T13:17:13,870 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41495-0x101a6d1666f0003, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:13,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:13,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:13,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:13,895 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T13:17:13,898 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=88aaa70743e7,42815,1733404631821 2024-12-05T13:17:13,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:13,916 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41495-0x101a6d1666f0003, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:13,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:13,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:13,941 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T13:17:13,944 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=88aaa70743e7,42815,1733404631821 2024-12-05T13:17:13,950 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-05T13:17:13,984 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(746): ClusterId : d611f681-e8be-43c0-b7da-5bfaf5772f57 2024-12-05T13:17:13,984 INFO [RS:2;88aaa70743e7:41495 {}] regionserver.HRegionServer(746): ClusterId : d611f681-e8be-43c0-b7da-5bfaf5772f57 2024-12-05T13:17:13,986 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(746): ClusterId : d611f681-e8be-43c0-b7da-5bfaf5772f57 2024-12-05T13:17:13,987 DEBUG [RS:2;88aaa70743e7:41495 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T13:17:13,987 DEBUG [RS:0;88aaa70743e7:39293 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T13:17:13,987 DEBUG [RS:1;88aaa70743e7:36055 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T13:17:14,004 DEBUG [RS:2;88aaa70743e7:41495 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T13:17:14,004 DEBUG [RS:0;88aaa70743e7:39293 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T13:17:14,004 DEBUG [RS:0;88aaa70743e7:39293 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T13:17:14,004 DEBUG [RS:2;88aaa70743e7:41495 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T13:17:14,005 DEBUG [RS:1;88aaa70743e7:36055 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T13:17:14,005 DEBUG [RS:1;88aaa70743e7:36055 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T13:17:14,021 DEBUG [RS:2;88aaa70743e7:41495 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T13:17:14,022 DEBUG [RS:0;88aaa70743e7:39293 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T13:17:14,022 DEBUG [RS:2;88aaa70743e7:41495 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@426192b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=88aaa70743e7/172.17.0.3:0 2024-12-05T13:17:14,022 DEBUG [RS:0;88aaa70743e7:39293 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cc491ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=88aaa70743e7/172.17.0.3:0 2024-12-05T13:17:14,023 DEBUG [RS:1;88aaa70743e7:36055 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T13:17:14,024 DEBUG [RS:1;88aaa70743e7:36055 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c2bd986, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=88aaa70743e7/172.17.0.3:0 2024-12-05T13:17:14,032 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:14,036 DEBUG [RS:0;88aaa70743e7:39293 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;88aaa70743e7:39293 2024-12-05T13:17:14,039 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T13:17:14,039 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T13:17:14,039 DEBUG [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T13:17:14,039 DEBUG [RS:1;88aaa70743e7:36055 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;88aaa70743e7:36055 2024-12-05T13:17:14,039 DEBUG [RS:2;88aaa70743e7:41495 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;88aaa70743e7:41495 2024-12-05T13:17:14,039 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T13:17:14,039 INFO [RS:2;88aaa70743e7:41495 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T13:17:14,039 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T13:17:14,039 INFO [RS:2;88aaa70743e7:41495 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T13:17:14,040 DEBUG [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T13:17:14,040 DEBUG [RS:2;88aaa70743e7:41495 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T13:17:14,042 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(2659): reportForDuty to master=88aaa70743e7,42815,1733404631821 with port=39293, startcode=1733404632706 2024-12-05T13:17:14,042 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(2659): reportForDuty to master=88aaa70743e7,42815,1733404631821 with port=36055, startcode=1733404632807 2024-12-05T13:17:14,042 INFO [RS:2;88aaa70743e7:41495 {}] regionserver.HRegionServer(2659): reportForDuty to master=88aaa70743e7,42815,1733404631821 with port=41495, startcode=1733404632848 2024-12-05T13:17:14,046 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-05T13:17:14,057 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T13:17:14,058 DEBUG [RS:1;88aaa70743e7:36055 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T13:17:14,059 DEBUG [RS:0;88aaa70743e7:39293 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T13:17:14,059 DEBUG [RS:2;88aaa70743e7:41495 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T13:17:14,065 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 88aaa70743e7,42815,1733404631821 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T13:17:14,076 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/88aaa70743e7:0, corePoolSize=5, maxPoolSize=5 2024-12-05T13:17:14,076 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/88aaa70743e7:0, corePoolSize=5, maxPoolSize=5 2024-12-05T13:17:14,077 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/88aaa70743e7:0, corePoolSize=5, maxPoolSize=5 2024-12-05T13:17:14,077 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/88aaa70743e7:0, corePoolSize=5, maxPoolSize=5 2024-12-05T13:17:14,077 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/88aaa70743e7:0, corePoolSize=10, maxPoolSize=10 2024-12-05T13:17:14,077 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,078 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/88aaa70743e7:0, corePoolSize=2, maxPoolSize=2 2024-12-05T13:17:14,078 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,099 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404664099 2024-12-05T13:17:14,101 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T13:17:14,102 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T13:17:14,104 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:14,105 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-05T13:17:14,109 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33729, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T13:17:14,109 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53729, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T13:17:14,109 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41317, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T13:17:14,109 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T13:17:14,110 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T13:17:14,110 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T13:17:14,110 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T13:17:14,112 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:14,112 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T13:17:14,111 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,118 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:14,119 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T13:17:14,120 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T13:17:14,121 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T13:17:14,123 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:14,124 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:14,139 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T13:17:14,139 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T13:17:14,148 DEBUG [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-05T13:17:14,148 DEBUG [RS:2;88aaa70743e7:41495 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-05T13:17:14,148 DEBUG [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-05T13:17:14,149 WARN [RS:2;88aaa70743e7:41495 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-05T13:17:14,149 WARN [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-05T13:17:14,149 WARN [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-05T13:17:14,149 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/88aaa70743e7:0:becomeActiveMaster-HFileCleaner.large.0-1733404634141,5,FailOnTimeoutGroup] 2024-12-05T13:17:14,149 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/88aaa70743e7:0:becomeActiveMaster-HFileCleaner.small.0-1733404634149,5,FailOnTimeoutGroup] 2024-12-05T13:17:14,150 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,150 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T13:17:14,151 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,152 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741831_1007 (size=1321) 2024-12-05T13:17:14,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741831_1007 (size=1321) 2024-12-05T13:17:14,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741831_1007 (size=1321) 2024-12-05T13:17:14,155 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-05T13:17:14,156 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:14,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741832_1008 (size=32) 2024-12-05T13:17:14,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741832_1008 (size=32) 2024-12-05T13:17:14,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741832_1008 (size=32) 2024-12-05T13:17:14,173 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:14,176 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T13:17:14,179 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T13:17:14,179 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:14,180 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:17:14,181 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T13:17:14,183 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T13:17:14,183 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:14,185 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:17:14,186 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T13:17:14,190 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T13:17:14,190 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:14,191 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:17:14,192 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T13:17:14,195 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T13:17:14,195 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:14,196 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:17:14,196 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T13:17:14,198 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740 2024-12-05T13:17:14,199 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740 2024-12-05T13:17:14,201 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T13:17:14,202 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T13:17:14,203 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T13:17:14,206 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T13:17:14,223 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:14,224 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60129409, jitterRate=-0.104001984000206}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T13:17:14,227 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733404634173Initializing all the Stores at 1733404634175 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733404634175Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733404634175Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404634176 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733404634176Cleaning up temporary data from old regions at 1733404634202 (+26 ms)Region opened successfully at 1733404634227 (+25 ms) 2024-12-05T13:17:14,228 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T13:17:14,228 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T13:17:14,228 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T13:17:14,228 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T13:17:14,228 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T13:17:14,236 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T13:17:14,236 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733404634228Disabling compacts and flushes for region at 1733404634228Disabling writes for close at 1733404634228Writing region close event to WAL at 1733404634235 (+7 ms)Closed at 1733404634236 (+1 ms) 2024-12-05T13:17:14,241 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:14,242 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-05T13:17:14,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T13:17:14,251 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(2659): reportForDuty to master=88aaa70743e7,42815,1733404631821 with port=36055, startcode=1733404632807 2024-12-05T13:17:14,255 INFO [RS:2;88aaa70743e7:41495 {}] regionserver.HRegionServer(2659): reportForDuty to master=88aaa70743e7,42815,1733404631821 with port=41495, startcode=1733404632848 2024-12-05T13:17:14,255 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(2659): reportForDuty to master=88aaa70743e7,42815,1733404631821 with port=39293, startcode=1733404632706 2024-12-05T13:17:14,258 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 88aaa70743e7,36055,1733404632807 2024-12-05T13:17:14,261 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] master.ServerManager(517): Registering regionserver=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:14,267 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:14,271 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T13:17:14,274 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 88aaa70743e7,39293,1733404632706 2024-12-05T13:17:14,274 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] master.ServerManager(517): Registering regionserver=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:14,274 DEBUG [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:14,274 DEBUG [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43169 2024-12-05T13:17:14,274 DEBUG [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T13:17:14,281 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 88aaa70743e7,41495,1733404632848 2024-12-05T13:17:14,281 DEBUG [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:14,281 DEBUG [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43169 2024-12-05T13:17:14,281 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] master.ServerManager(517): Registering regionserver=88aaa70743e7,41495,1733404632848 2024-12-05T13:17:14,281 DEBUG [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T13:17:14,285 DEBUG [RS:2;88aaa70743e7:41495 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:14,285 DEBUG [RS:2;88aaa70743e7:41495 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43169 2024-12-05T13:17:14,285 DEBUG [RS:2;88aaa70743e7:41495 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T13:17:14,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T13:17:14,337 DEBUG [RS:1;88aaa70743e7:36055 {}] zookeeper.ZKUtil(111): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/88aaa70743e7,36055,1733404632807 2024-12-05T13:17:14,337 DEBUG [RS:0;88aaa70743e7:39293 {}] zookeeper.ZKUtil(111): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/88aaa70743e7,39293,1733404632706 2024-12-05T13:17:14,337 WARN [RS:0;88aaa70743e7:39293 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T13:17:14,337 WARN [RS:1;88aaa70743e7:36055 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T13:17:14,337 INFO [RS:1;88aaa70743e7:36055 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:17:14,337 INFO [RS:0;88aaa70743e7:39293 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:17:14,338 DEBUG [RS:2;88aaa70743e7:41495 {}] zookeeper.ZKUtil(111): regionserver:41495-0x101a6d1666f0003, quorum=127.0.0.1:50894, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/88aaa70743e7,41495,1733404632848 2024-12-05T13:17:14,338 DEBUG [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,36055,1733404632807 2024-12-05T13:17:14,338 DEBUG [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,39293,1733404632706 2024-12-05T13:17:14,338 WARN [RS:2;88aaa70743e7:41495 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T13:17:14,338 INFO [RS:2;88aaa70743e7:41495 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:17:14,338 DEBUG [RS:2;88aaa70743e7:41495 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848 2024-12-05T13:17:14,339 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [88aaa70743e7,39293,1733404632706] 2024-12-05T13:17:14,340 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [88aaa70743e7,36055,1733404632807] 2024-12-05T13:17:14,340 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [88aaa70743e7,41495,1733404632848] 2024-12-05T13:17:14,371 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T13:17:14,371 INFO [RS:2;88aaa70743e7:41495 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T13:17:14,371 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T13:17:14,390 INFO [RS:2;88aaa70743e7:41495 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T13:17:14,390 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T13:17:14,390 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T13:17:14,397 INFO [RS:2;88aaa70743e7:41495 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T13:17:14,397 INFO [RS:1;88aaa70743e7:36055 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T13:17:14,396 INFO [RS:0;88aaa70743e7:39293 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T13:17:14,397 INFO [RS:2;88aaa70743e7:41495 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,397 INFO [RS:0;88aaa70743e7:39293 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,397 INFO [RS:1;88aaa70743e7:36055 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,398 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T13:17:14,398 INFO [RS:2;88aaa70743e7:41495 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T13:17:14,399 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T13:17:14,407 INFO [RS:0;88aaa70743e7:39293 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T13:17:14,408 INFO [RS:2;88aaa70743e7:41495 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T13:17:14,409 INFO [RS:1;88aaa70743e7:36055 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T13:17:14,409 INFO [RS:0;88aaa70743e7:39293 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,409 INFO [RS:1;88aaa70743e7:36055 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,409 INFO [RS:2;88aaa70743e7:41495 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,409 DEBUG [RS:0;88aaa70743e7:39293 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,409 DEBUG [RS:1;88aaa70743e7:36055 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,409 DEBUG [RS:2;88aaa70743e7:41495 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,410 DEBUG [RS:0;88aaa70743e7:39293 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,410 DEBUG [RS:1;88aaa70743e7:36055 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,410 DEBUG [RS:0;88aaa70743e7:39293 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,410 DEBUG [RS:2;88aaa70743e7:41495 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,410 DEBUG [RS:1;88aaa70743e7:36055 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,410 DEBUG [RS:0;88aaa70743e7:39293 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,410 DEBUG [RS:1;88aaa70743e7:36055 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,410 DEBUG [RS:0;88aaa70743e7:39293 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,410 DEBUG [RS:2;88aaa70743e7:41495 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,410 DEBUG [RS:1;88aaa70743e7:36055 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,411 DEBUG [RS:0;88aaa70743e7:39293 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0, corePoolSize=2, maxPoolSize=2 2024-12-05T13:17:14,411 DEBUG [RS:1;88aaa70743e7:36055 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0, corePoolSize=2, maxPoolSize=2 2024-12-05T13:17:14,411 DEBUG [RS:0;88aaa70743e7:39293 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,411 DEBUG [RS:2;88aaa70743e7:41495 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,411 DEBUG [RS:0;88aaa70743e7:39293 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,411 DEBUG [RS:1;88aaa70743e7:36055 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,411 DEBUG [RS:0;88aaa70743e7:39293 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,411 DEBUG [RS:1;88aaa70743e7:36055 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,411 DEBUG [RS:0;88aaa70743e7:39293 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,411 DEBUG [RS:1;88aaa70743e7:36055 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,411 DEBUG [RS:0;88aaa70743e7:39293 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,411 DEBUG [RS:0;88aaa70743e7:39293 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,411 DEBUG [RS:1;88aaa70743e7:36055 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,411 DEBUG [RS:0;88aaa70743e7:39293 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/88aaa70743e7:0, corePoolSize=3, maxPoolSize=3 2024-12-05T13:17:14,411 DEBUG [RS:1;88aaa70743e7:36055 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,411 DEBUG [RS:1;88aaa70743e7:36055 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,411 DEBUG [RS:0;88aaa70743e7:39293 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/88aaa70743e7:0, corePoolSize=3, maxPoolSize=3 2024-12-05T13:17:14,412 DEBUG [RS:2;88aaa70743e7:41495 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,412 DEBUG [RS:1;88aaa70743e7:36055 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/88aaa70743e7:0, corePoolSize=3, maxPoolSize=3 2024-12-05T13:17:14,412 DEBUG [RS:1;88aaa70743e7:36055 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/88aaa70743e7:0, corePoolSize=3, maxPoolSize=3 2024-12-05T13:17:14,412 DEBUG [RS:2;88aaa70743e7:41495 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0, corePoolSize=2, maxPoolSize=2 2024-12-05T13:17:14,412 DEBUG [RS:2;88aaa70743e7:41495 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,412 DEBUG [RS:2;88aaa70743e7:41495 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,412 DEBUG [RS:2;88aaa70743e7:41495 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,412 DEBUG [RS:2;88aaa70743e7:41495 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,412 DEBUG [RS:2;88aaa70743e7:41495 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,413 DEBUG [RS:2;88aaa70743e7:41495 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/88aaa70743e7:0, corePoolSize=1, maxPoolSize=1 2024-12-05T13:17:14,413 DEBUG [RS:2;88aaa70743e7:41495 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/88aaa70743e7:0, corePoolSize=3, maxPoolSize=3 2024-12-05T13:17:14,413 DEBUG [RS:2;88aaa70743e7:41495 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/88aaa70743e7:0, corePoolSize=3, maxPoolSize=3 2024-12-05T13:17:14,415 INFO [RS:0;88aaa70743e7:39293 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,415 INFO [RS:1;88aaa70743e7:36055 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,415 INFO [RS:0;88aaa70743e7:39293 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,415 INFO [RS:1;88aaa70743e7:36055 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,415 INFO [RS:0;88aaa70743e7:39293 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,415 INFO [RS:1;88aaa70743e7:36055 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,415 INFO [RS:0;88aaa70743e7:39293 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,415 INFO [RS:1;88aaa70743e7:36055 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,415 INFO [RS:0;88aaa70743e7:39293 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,415 INFO [RS:1;88aaa70743e7:36055 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,415 INFO [RS:0;88aaa70743e7:39293 {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,39293,1733404632706-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T13:17:14,416 INFO [RS:1;88aaa70743e7:36055 {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,36055,1733404632807-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T13:17:14,416 INFO [RS:2;88aaa70743e7:41495 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,416 INFO [RS:2;88aaa70743e7:41495 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,416 INFO [RS:2;88aaa70743e7:41495 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,416 INFO [RS:2;88aaa70743e7:41495 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,417 INFO [RS:2;88aaa70743e7:41495 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,417 INFO [RS:2;88aaa70743e7:41495 {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,41495,1733404632848-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T13:17:14,424 WARN [88aaa70743e7:42815 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T13:17:14,438 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T13:17:14,439 INFO [RS:2;88aaa70743e7:41495 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T13:17:14,440 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T13:17:14,441 INFO [RS:1;88aaa70743e7:36055 {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,36055,1733404632807-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,441 INFO [RS:2;88aaa70743e7:41495 {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,41495,1733404632848-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,441 INFO [RS:0;88aaa70743e7:39293 {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,39293,1733404632706-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,441 INFO [RS:2;88aaa70743e7:41495 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,441 INFO [RS:0;88aaa70743e7:39293 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,441 INFO [RS:2;88aaa70743e7:41495 {}] regionserver.Replication(171): 88aaa70743e7,41495,1733404632848 started 2024-12-05T13:17:14,441 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.Replication(171): 88aaa70743e7,39293,1733404632706 started 2024-12-05T13:17:14,447 INFO [RS:1;88aaa70743e7:36055 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,447 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.Replication(171): 88aaa70743e7,36055,1733404632807 started 2024-12-05T13:17:14,458 INFO [RS:2;88aaa70743e7:41495 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,458 INFO [RS:2;88aaa70743e7:41495 {}] regionserver.HRegionServer(1482): Serving as 88aaa70743e7,41495,1733404632848, RpcServer on 88aaa70743e7/172.17.0.3:41495, sessionid=0x101a6d1666f0003 2024-12-05T13:17:14,459 DEBUG [RS:2;88aaa70743e7:41495 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T13:17:14,459 DEBUG [RS:2;88aaa70743e7:41495 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 88aaa70743e7,41495,1733404632848 2024-12-05T13:17:14,459 DEBUG [RS:2;88aaa70743e7:41495 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88aaa70743e7,41495,1733404632848' 2024-12-05T13:17:14,459 DEBUG [RS:2;88aaa70743e7:41495 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T13:17:14,460 DEBUG [RS:2;88aaa70743e7:41495 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T13:17:14,460 DEBUG [RS:2;88aaa70743e7:41495 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T13:17:14,461 DEBUG [RS:2;88aaa70743e7:41495 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T13:17:14,461 DEBUG [RS:2;88aaa70743e7:41495 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 88aaa70743e7,41495,1733404632848 2024-12-05T13:17:14,461 DEBUG [RS:2;88aaa70743e7:41495 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88aaa70743e7,41495,1733404632848' 2024-12-05T13:17:14,461 DEBUG [RS:2;88aaa70743e7:41495 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T13:17:14,461 DEBUG [RS:2;88aaa70743e7:41495 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T13:17:14,462 INFO [RS:0;88aaa70743e7:39293 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,462 DEBUG [RS:2;88aaa70743e7:41495 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T13:17:14,462 INFO [RS:2;88aaa70743e7:41495 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T13:17:14,462 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(1482): Serving as 88aaa70743e7,39293,1733404632706, RpcServer on 88aaa70743e7/172.17.0.3:39293, sessionid=0x101a6d1666f0001 2024-12-05T13:17:14,462 INFO [RS:2;88aaa70743e7:41495 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T13:17:14,462 DEBUG [RS:0;88aaa70743e7:39293 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T13:17:14,462 DEBUG [RS:0;88aaa70743e7:39293 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 88aaa70743e7,39293,1733404632706 2024-12-05T13:17:14,462 DEBUG [RS:0;88aaa70743e7:39293 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88aaa70743e7,39293,1733404632706' 2024-12-05T13:17:14,462 DEBUG [RS:0;88aaa70743e7:39293 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T13:17:14,463 DEBUG [RS:0;88aaa70743e7:39293 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T13:17:14,464 DEBUG [RS:0;88aaa70743e7:39293 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T13:17:14,464 DEBUG [RS:0;88aaa70743e7:39293 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T13:17:14,464 DEBUG [RS:0;88aaa70743e7:39293 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 88aaa70743e7,39293,1733404632706 2024-12-05T13:17:14,464 DEBUG [RS:0;88aaa70743e7:39293 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88aaa70743e7,39293,1733404632706' 2024-12-05T13:17:14,464 DEBUG [RS:0;88aaa70743e7:39293 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T13:17:14,465 DEBUG [RS:0;88aaa70743e7:39293 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T13:17:14,465 DEBUG [RS:0;88aaa70743e7:39293 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T13:17:14,465 INFO [RS:0;88aaa70743e7:39293 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T13:17:14,465 INFO [RS:0;88aaa70743e7:39293 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T13:17:14,466 INFO [RS:1;88aaa70743e7:36055 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:14,467 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(1482): Serving as 88aaa70743e7,36055,1733404632807, RpcServer on 88aaa70743e7/172.17.0.3:36055, sessionid=0x101a6d1666f0002 2024-12-05T13:17:14,467 DEBUG [RS:1;88aaa70743e7:36055 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T13:17:14,467 DEBUG [RS:1;88aaa70743e7:36055 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 88aaa70743e7,36055,1733404632807 2024-12-05T13:17:14,467 DEBUG [RS:1;88aaa70743e7:36055 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88aaa70743e7,36055,1733404632807' 2024-12-05T13:17:14,467 DEBUG [RS:1;88aaa70743e7:36055 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T13:17:14,468 DEBUG [RS:1;88aaa70743e7:36055 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T13:17:14,469 DEBUG [RS:1;88aaa70743e7:36055 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T13:17:14,469 DEBUG [RS:1;88aaa70743e7:36055 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T13:17:14,469 DEBUG [RS:1;88aaa70743e7:36055 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 88aaa70743e7,36055,1733404632807 2024-12-05T13:17:14,469 DEBUG [RS:1;88aaa70743e7:36055 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88aaa70743e7,36055,1733404632807' 2024-12-05T13:17:14,469 DEBUG [RS:1;88aaa70743e7:36055 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T13:17:14,470 DEBUG [RS:1;88aaa70743e7:36055 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T13:17:14,470 DEBUG [RS:1;88aaa70743e7:36055 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T13:17:14,470 INFO [RS:1;88aaa70743e7:36055 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T13:17:14,470 INFO [RS:1;88aaa70743e7:36055 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T13:17:14,569 INFO [RS:0;88aaa70743e7:39293 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T13:17:14,569 INFO [RS:2;88aaa70743e7:41495 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T13:17:14,571 INFO [RS:1;88aaa70743e7:36055 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T13:17:14,572 INFO [RS:0;88aaa70743e7:39293 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=88aaa70743e7%2C39293%2C1733404632706, suffix=, logDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,39293,1733404632706, archiveDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/oldWALs, maxLogs=32 2024-12-05T13:17:14,572 INFO [RS:2;88aaa70743e7:41495 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=88aaa70743e7%2C41495%2C1733404632848, suffix=, logDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848, archiveDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/oldWALs, maxLogs=32 2024-12-05T13:17:14,574 INFO [RS:1;88aaa70743e7:36055 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=88aaa70743e7%2C36055%2C1733404632807, suffix=, logDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,36055,1733404632807, archiveDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/oldWALs, maxLogs=32 2024-12-05T13:17:14,589 DEBUG [RS:1;88aaa70743e7:36055 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,36055,1733404632807/88aaa70743e7%2C36055%2C1733404632807.1733404634576, exclude list is [], retry=0 2024-12-05T13:17:14,594 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42229,DS-0c343192-22cb-4379-b3b1-2cf2bdfac1b3,DISK] 2024-12-05T13:17:14,594 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32831,DS-da32b628-72dd-49f3-b291-1fa8797500ab,DISK] 2024-12-05T13:17:14,594 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34109,DS-d2ff1f2a-33de-4af6-9d9e-1d533285e26a,DISK] 2024-12-05T13:17:14,594 DEBUG [RS:2;88aaa70743e7:41495 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848/88aaa70743e7%2C41495%2C1733404632848.1733404634576, exclude list is [], retry=0 2024-12-05T13:17:14,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42229,DS-0c343192-22cb-4379-b3b1-2cf2bdfac1b3,DISK] 2024-12-05T13:17:14,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34109,DS-d2ff1f2a-33de-4af6-9d9e-1d533285e26a,DISK] 2024-12-05T13:17:14,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32831,DS-da32b628-72dd-49f3-b291-1fa8797500ab,DISK] 2024-12-05T13:17:14,637 DEBUG [RS:0;88aaa70743e7:39293 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,39293,1733404632706/88aaa70743e7%2C39293%2C1733404632706.1733404634577, exclude list is [], retry=0 2024-12-05T13:17:14,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42229,DS-0c343192-22cb-4379-b3b1-2cf2bdfac1b3,DISK] 2024-12-05T13:17:14,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34109,DS-d2ff1f2a-33de-4af6-9d9e-1d533285e26a,DISK] 2024-12-05T13:17:14,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32831,DS-da32b628-72dd-49f3-b291-1fa8797500ab,DISK] 2024-12-05T13:17:14,653 INFO [RS:1;88aaa70743e7:36055 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,36055,1733404632807/88aaa70743e7%2C36055%2C1733404632807.1733404634576 2024-12-05T13:17:14,659 DEBUG [RS:1;88aaa70743e7:36055 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34519:34519),(127.0.0.1/127.0.0.1:33987:33987),(127.0.0.1/127.0.0.1:40917:40917)] 2024-12-05T13:17:14,663 INFO [RS:2;88aaa70743e7:41495 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848/88aaa70743e7%2C41495%2C1733404632848.1733404634576 2024-12-05T13:17:14,666 DEBUG [RS:2;88aaa70743e7:41495 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34519:34519),(127.0.0.1/127.0.0.1:40917:40917),(127.0.0.1/127.0.0.1:33987:33987)] 2024-12-05T13:17:14,671 INFO [RS:0;88aaa70743e7:39293 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,39293,1733404632706/88aaa70743e7%2C39293%2C1733404632706.1733404634577 2024-12-05T13:17:14,671 DEBUG [RS:0;88aaa70743e7:39293 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33987:33987),(127.0.0.1/127.0.0.1:40917:40917),(127.0.0.1/127.0.0.1:34519:34519)] 2024-12-05T13:17:14,927 DEBUG [88aaa70743e7:42815 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-05T13:17:14,934 DEBUG [88aaa70743e7:42815 {}] balancer.BalancerClusterState(204): Hosts are {88aaa70743e7=0} racks are {/default-rack=0} 2024-12-05T13:17:14,941 DEBUG [88aaa70743e7:42815 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T13:17:14,941 DEBUG [88aaa70743e7:42815 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T13:17:14,941 DEBUG [88aaa70743e7:42815 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-05T13:17:14,941 DEBUG [88aaa70743e7:42815 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T13:17:14,941 DEBUG [88aaa70743e7:42815 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T13:17:14,941 DEBUG [88aaa70743e7:42815 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-05T13:17:14,941 INFO [88aaa70743e7:42815 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T13:17:14,941 INFO [88aaa70743e7:42815 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T13:17:14,941 INFO [88aaa70743e7:42815 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-05T13:17:14,941 DEBUG [88aaa70743e7:42815 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T13:17:14,949 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=88aaa70743e7,41495,1733404632848 2024-12-05T13:17:14,956 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,41495,1733404632848, state=OPENING 2024-12-05T13:17:15,017 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T13:17:15,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:15,025 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41495-0x101a6d1666f0003, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:15,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:15,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:15,026 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:15,026 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:15,026 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:15,026 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:15,027 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:15,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,41495,1733404632848}] 2024-12-05T13:17:15,208 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T13:17:15,210 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43553, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T13:17:15,221 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T13:17:15,222 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:17:15,222 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T13:17:15,226 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=88aaa70743e7%2C41495%2C1733404632848.meta, suffix=.meta, logDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848, archiveDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/oldWALs, maxLogs=32 2024-12-05T13:17:15,246 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848/88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, exclude list is [], retry=0 2024-12-05T13:17:15,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42229,DS-0c343192-22cb-4379-b3b1-2cf2bdfac1b3,DISK] 2024-12-05T13:17:15,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34109,DS-d2ff1f2a-33de-4af6-9d9e-1d533285e26a,DISK] 2024-12-05T13:17:15,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32831,DS-da32b628-72dd-49f3-b291-1fa8797500ab,DISK] 2024-12-05T13:17:15,255 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848/88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta 2024-12-05T13:17:15,256 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40917:40917),(127.0.0.1/127.0.0.1:33987:33987),(127.0.0.1/127.0.0.1:34519:34519)] 2024-12-05T13:17:15,256 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:17:15,258 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T13:17:15,261 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T13:17:15,265 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T13:17:15,269 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T13:17:15,270 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:15,270 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T13:17:15,270 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T13:17:15,273 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T13:17:15,276 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T13:17:15,276 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:15,277 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:17:15,277 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T13:17:15,279 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T13:17:15,279 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:15,281 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:17:15,281 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T13:17:15,284 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T13:17:15,284 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:15,286 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:17:15,286 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T13:17:15,288 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T13:17:15,288 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:15,289 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:17:15,290 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T13:17:15,291 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740 2024-12-05T13:17:15,294 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740 2024-12-05T13:17:15,297 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T13:17:15,297 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T13:17:15,298 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T13:17:15,303 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T13:17:15,305 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74541953, jitterRate=0.11076165735721588}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T13:17:15,305 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T13:17:15,306 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733404635270Writing region info on filesystem at 1733404635270Initializing all the Stores at 1733404635273 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733404635273Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733404635273Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404635273Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733404635273Cleaning up temporary data from old regions at 1733404635297 (+24 ms)Running coprocessor post-open hooks at 1733404635305 (+8 ms)Region opened successfully at 1733404635306 (+1 ms) 2024-12-05T13:17:15,316 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733404635199 2024-12-05T13:17:15,344 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,41495,1733404632848 2024-12-05T13:17:15,347 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,41495,1733404632848, state=OPEN 2024-12-05T13:17:15,348 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T13:17:15,349 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T13:17:15,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:15,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:15,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:15,359 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:15,359 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:15,359 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:15,359 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41495-0x101a6d1666f0003, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:15,359 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:15,360 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=88aaa70743e7,41495,1733404632848 2024-12-05T13:17:15,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T13:17:15,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,41495,1733404632848 in 331 msec 2024-12-05T13:17:15,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T13:17:15,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.1220 sec 2024-12-05T13:17:15,387 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:15,388 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-05T13:17:15,427 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T13:17:15,429 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,41495,1733404632848, seqNum=-1] 2024-12-05T13:17:15,458 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T13:17:15,461 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37317, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T13:17:15,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5070 sec 2024-12-05T13:17:15,491 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733404635491, completionTime=-1 2024-12-05T13:17:15,495 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-05T13:17:15,496 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:15,539 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-05T13:17:15,539 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404695539 2024-12-05T13:17:15,539 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404755539 2024-12-05T13:17:15,539 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 43 msec 2024-12-05T13:17:15,542 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:15,552 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,42815,1733404631821-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:15,552 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,42815,1733404631821-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:15,552 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,42815,1733404631821-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:15,554 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-88aaa70743e7:42815, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:15,555 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:15,559 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:15,564 DEBUG [master/88aaa70743e7:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-05T13:17:15,601 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.648sec 2024-12-05T13:17:15,604 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T13:17:15,607 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T13:17:15,608 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T13:17:15,609 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T13:17:15,614 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T13:17:15,616 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,42815,1733404631821-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T13:17:15,616 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,42815,1733404631821-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T13:17:15,621 DEBUG [master/88aaa70743e7:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-05T13:17:15,623 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T13:17:15,623 INFO [master/88aaa70743e7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88aaa70743e7,42815,1733404631821-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T13:17:15,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b3568d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T13:17:15,702 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-05T13:17:15,702 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-05T13:17:15,706 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 88aaa70743e7,42815,-1 for getting cluster id 2024-12-05T13:17:15,709 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T13:17:15,723 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd611f681-e8be-43c0-b7da-5bfaf5772f57' 2024-12-05T13:17:15,725 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T13:17:15,726 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d611f681-e8be-43c0-b7da-5bfaf5772f57" 2024-12-05T13:17:15,726 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@154288de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T13:17:15,726 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [88aaa70743e7,42815,-1] 2024-12-05T13:17:15,730 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T13:17:15,734 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T13:17:15,735 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T13:17:15,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d28ae4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T13:17:15,742 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T13:17:15,751 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,41495,1733404632848, seqNum=-1] 2024-12-05T13:17:15,751 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T13:17:15,754 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T13:17:15,776 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=88aaa70743e7,42815,1733404631821 2024-12-05T13:17:15,780 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T13:17:15,789 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 88aaa70743e7,42815,1733404631821 2024-12-05T13:17:15,830 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@52aea2af 2024-12-05T13:17:15,832 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T13:17:15,842 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T13:17:15,861 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T13:17:15,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:15,882 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T13:17:15,890 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-12-05T13:17:15,891 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:15,894 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T13:17:15,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T13:17:15,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741837_1013 (size=358) 2024-12-05T13:17:15,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741837_1013 (size=358) 2024-12-05T13:17:15,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741837_1013 (size=358) 2024-12-05T13:17:16,000 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5c9639ed41e308974b4199260665db3c, NAME => 'test,,1733404635844.5c9639ed41e308974b4199260665db3c.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,002 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 943cec28fed0bc3f97c304818b208e3f, NAME => 'test,ccc,1733404635844.943cec28fed0bc3f97c304818b208e3f.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,003 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 8b3cc71809f62cd16a817f5a172f68ff, NAME => 'test,bbb,1733404635844.8b3cc71809f62cd16a817f5a172f68ff.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T13:17:16,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741838_1014 (size=42) 2024-12-05T13:17:16,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741838_1014 (size=42) 2024-12-05T13:17:16,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741839_1015 (size=45) 2024-12-05T13:17:16,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741838_1014 (size=42) 2024-12-05T13:17:16,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741839_1015 (size=45) 2024-12-05T13:17:16,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741839_1015 (size=45) 2024-12-05T13:17:16,029 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1733404635844.5c9639ed41e308974b4199260665db3c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,029 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 5c9639ed41e308974b4199260665db3c, disabling compactions & flushes 2024-12-05T13:17:16,029 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1733404635844.5c9639ed41e308974b4199260665db3c. 2024-12-05T13:17:16,029 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1733404635844.5c9639ed41e308974b4199260665db3c. 2024-12-05T13:17:16,029 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1733404635844.5c9639ed41e308974b4199260665db3c. after waiting 0 ms 2024-12-05T13:17:16,029 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1733404635844.5c9639ed41e308974b4199260665db3c. 2024-12-05T13:17:16,029 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1733404635844.943cec28fed0bc3f97c304818b208e3f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,029 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1733404635844.5c9639ed41e308974b4199260665db3c. 2024-12-05T13:17:16,029 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5c9639ed41e308974b4199260665db3c: Waiting for close lock at 1733404636029Disabling compacts and flushes for region at 1733404636029Disabling writes for close at 1733404636029Writing region close event to WAL at 1733404636029Closed at 1733404636029 2024-12-05T13:17:16,029 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 943cec28fed0bc3f97c304818b208e3f, disabling compactions & flushes 2024-12-05T13:17:16,029 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1733404635844.943cec28fed0bc3f97c304818b208e3f. 2024-12-05T13:17:16,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741840_1016 (size=45) 2024-12-05T13:17:16,030 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1733404635844.943cec28fed0bc3f97c304818b208e3f. 2024-12-05T13:17:16,030 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1733404635844.943cec28fed0bc3f97c304818b208e3f. after waiting 0 ms 2024-12-05T13:17:16,030 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f8705d72ae6f07bbf8fa8bfbf5a81d07, NAME => 'test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,030 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1733404635844.943cec28fed0bc3f97c304818b208e3f. 2024-12-05T13:17:16,030 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1733404635844.943cec28fed0bc3f97c304818b208e3f. 2024-12-05T13:17:16,030 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 943cec28fed0bc3f97c304818b208e3f: Waiting for close lock at 1733404636029Disabling compacts and flushes for region at 1733404636029Disabling writes for close at 1733404636030 (+1 ms)Writing region close event to WAL at 1733404636030Closed at 1733404636030 2024-12-05T13:17:16,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741840_1016 (size=45) 2024-12-05T13:17:16,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741840_1016 (size=45) 2024-12-05T13:17:16,031 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 8e77e301280f5f9879631fbf6a099514, NAME => 'test,eee,1733404635844.8e77e301280f5f9879631fbf6a099514.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,032 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1733404635844.8b3cc71809f62cd16a817f5a172f68ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,032 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 8b3cc71809f62cd16a817f5a172f68ff, disabling compactions & flushes 2024-12-05T13:17:16,032 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1733404635844.8b3cc71809f62cd16a817f5a172f68ff. 2024-12-05T13:17:16,032 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1733404635844.8b3cc71809f62cd16a817f5a172f68ff. 2024-12-05T13:17:16,032 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1733404635844.8b3cc71809f62cd16a817f5a172f68ff. after waiting 0 ms 2024-12-05T13:17:16,032 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1733404635844.8b3cc71809f62cd16a817f5a172f68ff. 2024-12-05T13:17:16,032 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1733404635844.8b3cc71809f62cd16a817f5a172f68ff. 2024-12-05T13:17:16,032 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 8b3cc71809f62cd16a817f5a172f68ff: Waiting for close lock at 1733404636032Disabling compacts and flushes for region at 1733404636032Disabling writes for close at 1733404636032Writing region close event to WAL at 1733404636032Closed at 1733404636032 2024-12-05T13:17:16,034 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1218265d16f004d3f7900dc1adf62958, NAME => 'test,fff,1733404635844.1218265d16f004d3f7900dc1adf62958.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741843_1019 (size=45) 2024-12-05T13:17:16,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741843_1019 (size=45) 2024-12-05T13:17:16,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741843_1019 (size=45) 2024-12-05T13:17:16,047 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,fff,1733404635844.1218265d16f004d3f7900dc1adf62958.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,047 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 1218265d16f004d3f7900dc1adf62958, disabling compactions & flushes 2024-12-05T13:17:16,047 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,fff,1733404635844.1218265d16f004d3f7900dc1adf62958. 2024-12-05T13:17:16,047 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1733404635844.1218265d16f004d3f7900dc1adf62958. 2024-12-05T13:17:16,047 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1733404635844.1218265d16f004d3f7900dc1adf62958. after waiting 0 ms 2024-12-05T13:17:16,047 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1733404635844.1218265d16f004d3f7900dc1adf62958. 2024-12-05T13:17:16,047 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,fff,1733404635844.1218265d16f004d3f7900dc1adf62958. 2024-12-05T13:17:16,047 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 1218265d16f004d3f7900dc1adf62958: Waiting for close lock at 1733404636047Disabling compacts and flushes for region at 1733404636047Disabling writes for close at 1733404636047Writing region close event to WAL at 1733404636047Closed at 1733404636047 2024-12-05T13:17:16,048 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 4debf2a1857c920c4d806684d7cb0807, NAME => 'test,ggg,1733404635844.4debf2a1857c920c4d806684d7cb0807.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741842_1018 (size=45) 2024-12-05T13:17:16,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741842_1018 (size=45) 2024-12-05T13:17:16,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741842_1018 (size=45) 2024-12-05T13:17:16,052 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1733404635844.8e77e301280f5f9879631fbf6a099514.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,052 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 8e77e301280f5f9879631fbf6a099514, disabling compactions & flushes 2024-12-05T13:17:16,052 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1733404635844.8e77e301280f5f9879631fbf6a099514. 2024-12-05T13:17:16,052 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733404635844.8e77e301280f5f9879631fbf6a099514. 2024-12-05T13:17:16,052 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733404635844.8e77e301280f5f9879631fbf6a099514. after waiting 0 ms 2024-12-05T13:17:16,052 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733404635844.8e77e301280f5f9879631fbf6a099514. 2024-12-05T13:17:16,052 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1733404635844.8e77e301280f5f9879631fbf6a099514. 2024-12-05T13:17:16,052 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 8e77e301280f5f9879631fbf6a099514: Waiting for close lock at 1733404636052Disabling compacts and flushes for region at 1733404636052Disabling writes for close at 1733404636052Writing region close event to WAL at 1733404636052Closed at 1733404636052 2024-12-05T13:17:16,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741841_1017 (size=45) 2024-12-05T13:17:16,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741841_1017 (size=45) 2024-12-05T13:17:16,053 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 7bf9cb128da6b1afbc569bd4b25eba2d, NAME => 'test,hhh,1733404635844.7bf9cb128da6b1afbc569bd4b25eba2d.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741841_1017 (size=45) 2024-12-05T13:17:16,059 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,059 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing f8705d72ae6f07bbf8fa8bfbf5a81d07, disabling compactions & flushes 2024-12-05T13:17:16,059 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07. 2024-12-05T13:17:16,059 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07. 2024-12-05T13:17:16,060 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07. after waiting 0 ms 2024-12-05T13:17:16,060 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07. 2024-12-05T13:17:16,060 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07. 2024-12-05T13:17:16,060 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for f8705d72ae6f07bbf8fa8bfbf5a81d07: Waiting for close lock at 1733404636059Disabling compacts and flushes for region at 1733404636059Disabling writes for close at 1733404636060 (+1 ms)Writing region close event to WAL at 1733404636060Closed at 1733404636060 2024-12-05T13:17:16,061 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 01ef88de67b81e433e4128dc0a819605, NAME => 'test,iii,1733404635844.01ef88de67b81e433e4128dc0a819605.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741844_1020 (size=45) 2024-12-05T13:17:16,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741844_1020 (size=45) 2024-12-05T13:17:16,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741844_1020 (size=45) 2024-12-05T13:17:16,071 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ggg,1733404635844.4debf2a1857c920c4d806684d7cb0807.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,072 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 4debf2a1857c920c4d806684d7cb0807, disabling compactions & flushes 2024-12-05T13:17:16,072 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ggg,1733404635844.4debf2a1857c920c4d806684d7cb0807. 2024-12-05T13:17:16,072 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1733404635844.4debf2a1857c920c4d806684d7cb0807. 2024-12-05T13:17:16,072 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1733404635844.4debf2a1857c920c4d806684d7cb0807. after waiting 0 ms 2024-12-05T13:17:16,072 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1733404635844.4debf2a1857c920c4d806684d7cb0807. 2024-12-05T13:17:16,072 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ggg,1733404635844.4debf2a1857c920c4d806684d7cb0807. 2024-12-05T13:17:16,072 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 4debf2a1857c920c4d806684d7cb0807: Waiting for close lock at 1733404636072Disabling compacts and flushes for region at 1733404636072Disabling writes for close at 1733404636072Writing region close event to WAL at 1733404636072Closed at 1733404636072 2024-12-05T13:17:16,073 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 8d9d0bbe2316236e0d9afb8d72cd4099, NAME => 'test,jjj,1733404635844.8d9d0bbe2316236e0d9afb8d72cd4099.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741845_1021 (size=45) 2024-12-05T13:17:16,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741845_1021 (size=45) 2024-12-05T13:17:16,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741845_1021 (size=45) 2024-12-05T13:17:16,079 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,hhh,1733404635844.7bf9cb128da6b1afbc569bd4b25eba2d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,079 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 7bf9cb128da6b1afbc569bd4b25eba2d, disabling compactions & flushes 2024-12-05T13:17:16,079 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,hhh,1733404635844.7bf9cb128da6b1afbc569bd4b25eba2d. 2024-12-05T13:17:16,080 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1733404635844.7bf9cb128da6b1afbc569bd4b25eba2d. 2024-12-05T13:17:16,080 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1733404635844.7bf9cb128da6b1afbc569bd4b25eba2d. after waiting 0 ms 2024-12-05T13:17:16,080 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1733404635844.7bf9cb128da6b1afbc569bd4b25eba2d. 2024-12-05T13:17:16,080 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,hhh,1733404635844.7bf9cb128da6b1afbc569bd4b25eba2d. 2024-12-05T13:17:16,080 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 7bf9cb128da6b1afbc569bd4b25eba2d: Waiting for close lock at 1733404636079Disabling compacts and flushes for region at 1733404636079Disabling writes for close at 1733404636080 (+1 ms)Writing region close event to WAL at 1733404636080Closed at 1733404636080 2024-12-05T13:17:16,081 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f29f7ad7d2427bd1681491cb1725214f, NAME => 'test,kkk,1733404635844.f29f7ad7d2427bd1681491cb1725214f.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741846_1022 (size=45) 2024-12-05T13:17:16,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741846_1022 (size=45) 2024-12-05T13:17:16,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741846_1022 (size=45) 2024-12-05T13:17:16,088 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,iii,1733404635844.01ef88de67b81e433e4128dc0a819605.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,088 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 01ef88de67b81e433e4128dc0a819605, disabling compactions & flushes 2024-12-05T13:17:16,088 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,iii,1733404635844.01ef88de67b81e433e4128dc0a819605. 2024-12-05T13:17:16,088 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1733404635844.01ef88de67b81e433e4128dc0a819605. 2024-12-05T13:17:16,088 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1733404635844.01ef88de67b81e433e4128dc0a819605. after waiting 0 ms 2024-12-05T13:17:16,088 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1733404635844.01ef88de67b81e433e4128dc0a819605. 2024-12-05T13:17:16,088 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,iii,1733404635844.01ef88de67b81e433e4128dc0a819605. 2024-12-05T13:17:16,088 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 01ef88de67b81e433e4128dc0a819605: Waiting for close lock at 1733404636088Disabling compacts and flushes for region at 1733404636088Disabling writes for close at 1733404636088Writing region close event to WAL at 1733404636088Closed at 1733404636088 2024-12-05T13:17:16,089 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3ea448375edac1598b6cdff464d477bf, NAME => 'test,lll,1733404635844.3ea448375edac1598b6cdff464d477bf.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741847_1023 (size=45) 2024-12-05T13:17:16,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741847_1023 (size=45) 2024-12-05T13:17:16,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741847_1023 (size=45) 2024-12-05T13:17:16,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741849_1025 (size=45) 2024-12-05T13:17:16,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741849_1025 (size=45) 2024-12-05T13:17:16,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741849_1025 (size=45) 2024-12-05T13:17:16,110 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,jjj,1733404635844.8d9d0bbe2316236e0d9afb8d72cd4099.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,110 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 8d9d0bbe2316236e0d9afb8d72cd4099, disabling compactions & flushes 2024-12-05T13:17:16,110 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,jjj,1733404635844.8d9d0bbe2316236e0d9afb8d72cd4099. 2024-12-05T13:17:16,110 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1733404635844.8d9d0bbe2316236e0d9afb8d72cd4099. 2024-12-05T13:17:16,110 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1733404635844.8d9d0bbe2316236e0d9afb8d72cd4099. after waiting 0 ms 2024-12-05T13:17:16,110 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1733404635844.8d9d0bbe2316236e0d9afb8d72cd4099. 2024-12-05T13:17:16,110 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,jjj,1733404635844.8d9d0bbe2316236e0d9afb8d72cd4099. 2024-12-05T13:17:16,110 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 8d9d0bbe2316236e0d9afb8d72cd4099: Waiting for close lock at 1733404636110Disabling compacts and flushes for region at 1733404636110Disabling writes for close at 1733404636110Writing region close event to WAL at 1733404636110Closed at 1733404636110 2024-12-05T13:17:16,111 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => eb2ca1f66d6c9b4513c79bf39e4e5bfe, NAME => 'test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741848_1024 (size=45) 2024-12-05T13:17:16,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741848_1024 (size=45) 2024-12-05T13:17:16,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741848_1024 (size=45) 2024-12-05T13:17:16,120 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,lll,1733404635844.3ea448375edac1598b6cdff464d477bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,120 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 3ea448375edac1598b6cdff464d477bf, disabling compactions & flushes 2024-12-05T13:17:16,120 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,lll,1733404635844.3ea448375edac1598b6cdff464d477bf. 2024-12-05T13:17:16,120 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1733404635844.3ea448375edac1598b6cdff464d477bf. 2024-12-05T13:17:16,120 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1733404635844.3ea448375edac1598b6cdff464d477bf. after waiting 0 ms 2024-12-05T13:17:16,120 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1733404635844.3ea448375edac1598b6cdff464d477bf. 2024-12-05T13:17:16,120 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,lll,1733404635844.3ea448375edac1598b6cdff464d477bf. 2024-12-05T13:17:16,120 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3ea448375edac1598b6cdff464d477bf: Waiting for close lock at 1733404636120Disabling compacts and flushes for region at 1733404636120Disabling writes for close at 1733404636120Writing region close event to WAL at 1733404636120Closed at 1733404636120 2024-12-05T13:17:16,121 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c086578586a3011e8fde7b2976cca84b, NAME => 'test,nnn,1733404635844.c086578586a3011e8fde7b2976cca84b.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741850_1026 (size=45) 2024-12-05T13:17:16,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741850_1026 (size=45) 2024-12-05T13:17:16,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741850_1026 (size=45) 2024-12-05T13:17:16,135 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,135 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing eb2ca1f66d6c9b4513c79bf39e4e5bfe, disabling compactions & flushes 2024-12-05T13:17:16,135 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe. 2024-12-05T13:17:16,135 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe. 2024-12-05T13:17:16,135 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe. after waiting 0 ms 2024-12-05T13:17:16,135 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe. 2024-12-05T13:17:16,135 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe. 2024-12-05T13:17:16,135 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for eb2ca1f66d6c9b4513c79bf39e4e5bfe: Waiting for close lock at 1733404636135Disabling compacts and flushes for region at 1733404636135Disabling writes for close at 1733404636135Writing region close event to WAL at 1733404636135Closed at 1733404636135 2024-12-05T13:17:16,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741851_1027 (size=45) 2024-12-05T13:17:16,136 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => bd586ec5ee08f004669e3c48942b3638, NAME => 'test,ooo,1733404635844.bd586ec5ee08f004669e3c48942b3638.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741851_1027 (size=45) 2024-12-05T13:17:16,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741851_1027 (size=45) 2024-12-05T13:17:16,137 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1733404635844.c086578586a3011e8fde7b2976cca84b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,137 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing c086578586a3011e8fde7b2976cca84b, disabling compactions & flushes 2024-12-05T13:17:16,137 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1733404635844.c086578586a3011e8fde7b2976cca84b. 2024-12-05T13:17:16,137 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1733404635844.c086578586a3011e8fde7b2976cca84b. 2024-12-05T13:17:16,138 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1733404635844.c086578586a3011e8fde7b2976cca84b. after waiting 0 ms 2024-12-05T13:17:16,138 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1733404635844.c086578586a3011e8fde7b2976cca84b. 2024-12-05T13:17:16,138 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1733404635844.c086578586a3011e8fde7b2976cca84b. 2024-12-05T13:17:16,138 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for c086578586a3011e8fde7b2976cca84b: Waiting for close lock at 1733404636137Disabling compacts and flushes for region at 1733404636137Disabling writes for close at 1733404636138 (+1 ms)Writing region close event to WAL at 1733404636138Closed at 1733404636138 2024-12-05T13:17:16,138 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3c03197a9f3300f90c023ccedb4f2112, NAME => 'test,ppp,1733404635844.3c03197a9f3300f90c023ccedb4f2112.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741852_1028 (size=45) 2024-12-05T13:17:16,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741852_1028 (size=45) 2024-12-05T13:17:16,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741852_1028 (size=45) 2024-12-05T13:17:16,152 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ooo,1733404635844.bd586ec5ee08f004669e3c48942b3638.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741853_1029 (size=45) 2024-12-05T13:17:16,152 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing bd586ec5ee08f004669e3c48942b3638, disabling compactions & flushes 2024-12-05T13:17:16,152 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ooo,1733404635844.bd586ec5ee08f004669e3c48942b3638. 2024-12-05T13:17:16,152 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1733404635844.bd586ec5ee08f004669e3c48942b3638. 2024-12-05T13:17:16,152 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1733404635844.bd586ec5ee08f004669e3c48942b3638. after waiting 0 ms 2024-12-05T13:17:16,153 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1733404635844.bd586ec5ee08f004669e3c48942b3638. 2024-12-05T13:17:16,153 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ooo,1733404635844.bd586ec5ee08f004669e3c48942b3638. 2024-12-05T13:17:16,153 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for bd586ec5ee08f004669e3c48942b3638: Waiting for close lock at 1733404636152Disabling compacts and flushes for region at 1733404636152Disabling writes for close at 1733404636153 (+1 ms)Writing region close event to WAL at 1733404636153Closed at 1733404636153 2024-12-05T13:17:16,153 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 5d414b07d03c5766c12220a94662f4ff, NAME => 'test,qqq,1733404635844.5d414b07d03c5766c12220a94662f4ff.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741853_1029 (size=45) 2024-12-05T13:17:16,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741853_1029 (size=45) 2024-12-05T13:17:16,154 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ppp,1733404635844.3c03197a9f3300f90c023ccedb4f2112.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,155 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 3c03197a9f3300f90c023ccedb4f2112, disabling compactions & flushes 2024-12-05T13:17:16,155 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ppp,1733404635844.3c03197a9f3300f90c023ccedb4f2112. 2024-12-05T13:17:16,155 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1733404635844.3c03197a9f3300f90c023ccedb4f2112. 2024-12-05T13:17:16,155 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1733404635844.3c03197a9f3300f90c023ccedb4f2112. after waiting 0 ms 2024-12-05T13:17:16,155 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1733404635844.3c03197a9f3300f90c023ccedb4f2112. 2024-12-05T13:17:16,155 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ppp,1733404635844.3c03197a9f3300f90c023ccedb4f2112. 2024-12-05T13:17:16,155 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3c03197a9f3300f90c023ccedb4f2112: Waiting for close lock at 1733404636155Disabling compacts and flushes for region at 1733404636155Disabling writes for close at 1733404636155Writing region close event to WAL at 1733404636155Closed at 1733404636155 2024-12-05T13:17:16,156 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a362f9452e2fdf5e9192ce69e889c30d, NAME => 'test,rrr,1733404635844.a362f9452e2fdf5e9192ce69e889c30d.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741854_1030 (size=45) 2024-12-05T13:17:16,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741855_1031 (size=45) 2024-12-05T13:17:16,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741854_1030 (size=45) 2024-12-05T13:17:16,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741855_1031 (size=45) 2024-12-05T13:17:16,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741854_1030 (size=45) 2024-12-05T13:17:16,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741855_1031 (size=45) 2024-12-05T13:17:16,175 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,rrr,1733404635844.a362f9452e2fdf5e9192ce69e889c30d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,175 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,qqq,1733404635844.5d414b07d03c5766c12220a94662f4ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,175 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 5d414b07d03c5766c12220a94662f4ff, disabling compactions & flushes 2024-12-05T13:17:16,175 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing a362f9452e2fdf5e9192ce69e889c30d, disabling compactions & flushes 2024-12-05T13:17:16,175 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,qqq,1733404635844.5d414b07d03c5766c12220a94662f4ff. 2024-12-05T13:17:16,175 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,rrr,1733404635844.a362f9452e2fdf5e9192ce69e889c30d. 2024-12-05T13:17:16,175 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1733404635844.5d414b07d03c5766c12220a94662f4ff. 2024-12-05T13:17:16,175 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1733404635844.a362f9452e2fdf5e9192ce69e889c30d. 2024-12-05T13:17:16,175 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1733404635844.5d414b07d03c5766c12220a94662f4ff. after waiting 0 ms 2024-12-05T13:17:16,175 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1733404635844.5d414b07d03c5766c12220a94662f4ff. 2024-12-05T13:17:16,175 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1733404635844.a362f9452e2fdf5e9192ce69e889c30d. after waiting 0 ms 2024-12-05T13:17:16,175 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,qqq,1733404635844.5d414b07d03c5766c12220a94662f4ff. 2024-12-05T13:17:16,175 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1733404635844.a362f9452e2fdf5e9192ce69e889c30d. 2024-12-05T13:17:16,175 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 5d414b07d03c5766c12220a94662f4ff: Waiting for close lock at 1733404636175Disabling compacts and flushes for region at 1733404636175Disabling writes for close at 1733404636175Writing region close event to WAL at 1733404636175Closed at 1733404636175 2024-12-05T13:17:16,175 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,rrr,1733404635844.a362f9452e2fdf5e9192ce69e889c30d. 2024-12-05T13:17:16,175 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for a362f9452e2fdf5e9192ce69e889c30d: Waiting for close lock at 1733404636175Disabling compacts and flushes for region at 1733404636175Disabling writes for close at 1733404636175Writing region close event to WAL at 1733404636175Closed at 1733404636175 2024-12-05T13:17:16,176 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 97426aa8ec496bfcaaf589a5cbe6f1b3, NAME => 'test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,176 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bb504ea56b81b22b5ca81b650f200ded, NAME => 'test,ttt,1733404635844.bb504ea56b81b22b5ca81b650f200ded.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741857_1033 (size=45) 2024-12-05T13:17:16,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741857_1033 (size=45) 2024-12-05T13:17:16,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741857_1033 (size=45) 2024-12-05T13:17:16,205 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,205 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 97426aa8ec496bfcaaf589a5cbe6f1b3, disabling compactions & flushes 2024-12-05T13:17:16,205 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3. 2024-12-05T13:17:16,205 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3. 2024-12-05T13:17:16,205 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3. after waiting 0 ms 2024-12-05T13:17:16,205 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3. 2024-12-05T13:17:16,205 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3. 2024-12-05T13:17:16,205 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 97426aa8ec496bfcaaf589a5cbe6f1b3: Waiting for close lock at 1733404636205Disabling compacts and flushes for region at 1733404636205Disabling writes for close at 1733404636205Writing region close event to WAL at 1733404636205Closed at 1733404636205 2024-12-05T13:17:16,207 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => f4b86b8085620be60c3a91043e8f4d72, NAME => 'test,uuu,1733404635844.f4b86b8085620be60c3a91043e8f4d72.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741856_1032 (size=45) 2024-12-05T13:17:16,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741856_1032 (size=45) 2024-12-05T13:17:16,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741856_1032 (size=45) 2024-12-05T13:17:16,209 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ttt,1733404635844.bb504ea56b81b22b5ca81b650f200ded.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,209 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing bb504ea56b81b22b5ca81b650f200ded, disabling compactions & flushes 2024-12-05T13:17:16,209 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ttt,1733404635844.bb504ea56b81b22b5ca81b650f200ded. 2024-12-05T13:17:16,209 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1733404635844.bb504ea56b81b22b5ca81b650f200ded. 2024-12-05T13:17:16,209 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1733404635844.bb504ea56b81b22b5ca81b650f200ded. after waiting 0 ms 2024-12-05T13:17:16,209 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1733404635844.bb504ea56b81b22b5ca81b650f200ded. 2024-12-05T13:17:16,209 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ttt,1733404635844.bb504ea56b81b22b5ca81b650f200ded. 2024-12-05T13:17:16,209 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for bb504ea56b81b22b5ca81b650f200ded: Waiting for close lock at 1733404636209Disabling compacts and flushes for region at 1733404636209Disabling writes for close at 1733404636209Writing region close event to WAL at 1733404636209Closed at 1733404636209 2024-12-05T13:17:16,210 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cb80ccb4b3541210c6fda8b4b988adb5, NAME => 'test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T13:17:16,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741858_1034 (size=45) 2024-12-05T13:17:16,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741858_1034 (size=45) 2024-12-05T13:17:16,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741858_1034 (size=45) 2024-12-05T13:17:16,227 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1733404635844.f4b86b8085620be60c3a91043e8f4d72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,227 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing f4b86b8085620be60c3a91043e8f4d72, disabling compactions & flushes 2024-12-05T13:17:16,227 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1733404635844.f4b86b8085620be60c3a91043e8f4d72. 2024-12-05T13:17:16,227 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1733404635844.f4b86b8085620be60c3a91043e8f4d72. 2024-12-05T13:17:16,227 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1733404635844.f4b86b8085620be60c3a91043e8f4d72. after waiting 0 ms 2024-12-05T13:17:16,227 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1733404635844.f4b86b8085620be60c3a91043e8f4d72. 2024-12-05T13:17:16,227 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1733404635844.f4b86b8085620be60c3a91043e8f4d72. 2024-12-05T13:17:16,227 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for f4b86b8085620be60c3a91043e8f4d72: Waiting for close lock at 1733404636227Disabling compacts and flushes for region at 1733404636227Disabling writes for close at 1733404636227Writing region close event to WAL at 1733404636227Closed at 1733404636227 2024-12-05T13:17:16,228 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => abced629a2bc26273048aa2f4b6f8d9c, NAME => 'test,www,1733404635844.abced629a2bc26273048aa2f4b6f8d9c.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741859_1035 (size=45) 2024-12-05T13:17:16,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741859_1035 (size=45) 2024-12-05T13:17:16,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741859_1035 (size=45) 2024-12-05T13:17:16,230 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,230 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing cb80ccb4b3541210c6fda8b4b988adb5, disabling compactions & flushes 2024-12-05T13:17:16,230 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5. 2024-12-05T13:17:16,230 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5. 2024-12-05T13:17:16,231 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5. after waiting 0 ms 2024-12-05T13:17:16,231 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5. 2024-12-05T13:17:16,231 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5. 2024-12-05T13:17:16,231 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for cb80ccb4b3541210c6fda8b4b988adb5: Waiting for close lock at 1733404636230Disabling compacts and flushes for region at 1733404636230Disabling writes for close at 1733404636231 (+1 ms)Writing region close event to WAL at 1733404636231Closed at 1733404636231 2024-12-05T13:17:16,231 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e805a0311eea683702ecc957a61057e2, NAME => 'test,xxx,1733404635844.e805a0311eea683702ecc957a61057e2.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741860_1036 (size=45) 2024-12-05T13:17:16,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741860_1036 (size=45) 2024-12-05T13:17:16,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741860_1036 (size=45) 2024-12-05T13:17:16,241 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,www,1733404635844.abced629a2bc26273048aa2f4b6f8d9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,241 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing abced629a2bc26273048aa2f4b6f8d9c, disabling compactions & flushes 2024-12-05T13:17:16,241 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,www,1733404635844.abced629a2bc26273048aa2f4b6f8d9c. 2024-12-05T13:17:16,241 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1733404635844.abced629a2bc26273048aa2f4b6f8d9c. 2024-12-05T13:17:16,241 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1733404635844.abced629a2bc26273048aa2f4b6f8d9c. after waiting 0 ms 2024-12-05T13:17:16,241 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1733404635844.abced629a2bc26273048aa2f4b6f8d9c. 2024-12-05T13:17:16,241 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,www,1733404635844.abced629a2bc26273048aa2f4b6f8d9c. 2024-12-05T13:17:16,241 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for abced629a2bc26273048aa2f4b6f8d9c: Waiting for close lock at 1733404636241Disabling compacts and flushes for region at 1733404636241Disabling writes for close at 1733404636241Writing region close event to WAL at 1733404636241Closed at 1733404636241 2024-12-05T13:17:16,242 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => cabc686ba5a0c762c06d5a4d0024dd7b, NAME => 'test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741861_1037 (size=45) 2024-12-05T13:17:16,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741861_1037 (size=45) 2024-12-05T13:17:16,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741861_1037 (size=45) 2024-12-05T13:17:16,247 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,xxx,1733404635844.e805a0311eea683702ecc957a61057e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,247 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing e805a0311eea683702ecc957a61057e2, disabling compactions & flushes 2024-12-05T13:17:16,247 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,xxx,1733404635844.e805a0311eea683702ecc957a61057e2. 2024-12-05T13:17:16,248 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733404635844.e805a0311eea683702ecc957a61057e2. 2024-12-05T13:17:16,248 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733404635844.e805a0311eea683702ecc957a61057e2. after waiting 0 ms 2024-12-05T13:17:16,248 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733404635844.e805a0311eea683702ecc957a61057e2. 2024-12-05T13:17:16,248 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,xxx,1733404635844.e805a0311eea683702ecc957a61057e2. 2024-12-05T13:17:16,248 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for e805a0311eea683702ecc957a61057e2: Waiting for close lock at 1733404636247Disabling compacts and flushes for region at 1733404636247Disabling writes for close at 1733404636248 (+1 ms)Writing region close event to WAL at 1733404636248Closed at 1733404636248 2024-12-05T13:17:16,248 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6f220ce002468367c756923f6fbc1fc1, NAME => 'test,zzz,1733404635844.6f220ce002468367c756923f6fbc1fc1.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75 2024-12-05T13:17:16,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741862_1038 (size=45) 2024-12-05T13:17:16,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741862_1038 (size=45) 2024-12-05T13:17:16,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741862_1038 (size=45) 2024-12-05T13:17:16,254 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,255 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing cabc686ba5a0c762c06d5a4d0024dd7b, disabling compactions & flushes 2024-12-05T13:17:16,255 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b. 2024-12-05T13:17:16,255 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b. 2024-12-05T13:17:16,255 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b. after waiting 0 ms 2024-12-05T13:17:16,255 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b. 2024-12-05T13:17:16,255 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b. 2024-12-05T13:17:16,255 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for cabc686ba5a0c762c06d5a4d0024dd7b: Waiting for close lock at 1733404636255Disabling compacts and flushes for region at 1733404636255Disabling writes for close at 1733404636255Writing region close event to WAL at 1733404636255Closed at 1733404636255 2024-12-05T13:17:16,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741863_1039 (size=42) 2024-12-05T13:17:16,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741863_1039 (size=42) 2024-12-05T13:17:16,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741863_1039 (size=42) 2024-12-05T13:17:16,259 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,zzz,1733404635844.6f220ce002468367c756923f6fbc1fc1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,260 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 6f220ce002468367c756923f6fbc1fc1, disabling compactions & flushes 2024-12-05T13:17:16,260 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,zzz,1733404635844.6f220ce002468367c756923f6fbc1fc1. 2024-12-05T13:17:16,260 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1733404635844.6f220ce002468367c756923f6fbc1fc1. 2024-12-05T13:17:16,260 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1733404635844.6f220ce002468367c756923f6fbc1fc1. after waiting 0 ms 2024-12-05T13:17:16,260 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1733404635844.6f220ce002468367c756923f6fbc1fc1. 2024-12-05T13:17:16,260 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,zzz,1733404635844.6f220ce002468367c756923f6fbc1fc1. 2024-12-05T13:17:16,260 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6f220ce002468367c756923f6fbc1fc1: Waiting for close lock at 1733404636260Disabling compacts and flushes for region at 1733404636260Disabling writes for close at 1733404636260Writing region close event to WAL at 1733404636260Closed at 1733404636260 2024-12-05T13:17:16,505 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,kkk,1733404635844.f29f7ad7d2427bd1681491cb1725214f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,505 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f29f7ad7d2427bd1681491cb1725214f, disabling compactions & flushes 2024-12-05T13:17:16,505 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,kkk,1733404635844.f29f7ad7d2427bd1681491cb1725214f. 2024-12-05T13:17:16,505 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1733404635844.f29f7ad7d2427bd1681491cb1725214f. 2024-12-05T13:17:16,505 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1733404635844.f29f7ad7d2427bd1681491cb1725214f. after waiting 0 ms 2024-12-05T13:17:16,505 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1733404635844.f29f7ad7d2427bd1681491cb1725214f. 2024-12-05T13:17:16,505 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,kkk,1733404635844.f29f7ad7d2427bd1681491cb1725214f. 2024-12-05T13:17:16,505 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f29f7ad7d2427bd1681491cb1725214f: Waiting for close lock at 1733404636505Disabling compacts and flushes for region at 1733404636505Disabling writes for close at 1733404636505Writing region close event to WAL at 1733404636505Closed at 1733404636505 2024-12-05T13:17:16,509 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T13:17:16,517 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1733404635844.5c9639ed41e308974b4199260665db3c.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,517 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1733404635844.943cec28fed0bc3f97c304818b208e3f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,518 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1733404635844.8b3cc71809f62cd16a817f5a172f68ff.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,518 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1733404635844.1218265d16f004d3f7900dc1adf62958.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,518 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1733404635844.8e77e301280f5f9879631fbf6a099514.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,518 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,519 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1733404635844.4debf2a1857c920c4d806684d7cb0807.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,519 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1733404635844.7bf9cb128da6b1afbc569bd4b25eba2d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,519 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1733404635844.01ef88de67b81e433e4128dc0a819605.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,519 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1733404635844.8d9d0bbe2316236e0d9afb8d72cd4099.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,519 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1733404635844.3ea448375edac1598b6cdff464d477bf.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,519 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,520 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1733404635844.c086578586a3011e8fde7b2976cca84b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,520 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1733404635844.bd586ec5ee08f004669e3c48942b3638.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,520 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1733404635844.3c03197a9f3300f90c023ccedb4f2112.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,520 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1733404635844.5d414b07d03c5766c12220a94662f4ff.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,520 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1733404635844.a362f9452e2fdf5e9192ce69e889c30d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,520 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,521 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1733404635844.bb504ea56b81b22b5ca81b650f200ded.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,521 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1733404635844.f4b86b8085620be60c3a91043e8f4d72.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,521 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,521 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1733404635844.abced629a2bc26273048aa2f4b6f8d9c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,521 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1733404635844.e805a0311eea683702ecc957a61057e2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,521 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,521 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1733404635844.6f220ce002468367c756923f6fbc1fc1.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,521 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1733404635844.f29f7ad7d2427bd1681491cb1725214f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1733404636509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733404636509"}]},"ts":"1733404636509"} 2024-12-05T13:17:16,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T13:17:16,577 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-12-05T13:17:16,580 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T13:17:16,583 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733404636580"}]},"ts":"1733404636580"} 2024-12-05T13:17:16,588 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-12-05T13:17:16,589 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {88aaa70743e7=0} racks are {/default-rack=0} 2024-12-05T13:17:16,609 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T13:17:16,609 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T13:17:16,609 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-05T13:17:16,609 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T13:17:16,609 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T13:17:16,609 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-05T13:17:16,609 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T13:17:16,610 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T13:17:16,610 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-05T13:17:16,610 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T13:17:16,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5c9639ed41e308974b4199260665db3c, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=943cec28fed0bc3f97c304818b208e3f, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8e77e301280f5f9879631fbf6a099514, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1218265d16f004d3f7900dc1adf62958, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4debf2a1857c920c4d806684d7cb0807, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7bf9cb128da6b1afbc569bd4b25eba2d, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8d9d0bbe2316236e0d9afb8d72cd4099, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f29f7ad7d2427bd1681491cb1725214f, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3ea448375edac1598b6cdff464d477bf, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c086578586a3011e8fde7b2976cca84b, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bd586ec5ee08f004669e3c48942b3638, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5d414b07d03c5766c12220a94662f4ff, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a362f9452e2fdf5e9192ce69e889c30d, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bb504ea56b81b22b5ca81b650f200ded, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f4b86b8085620be60c3a91043e8f4d72, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=abced629a2bc26273048aa2f4b6f8d9c, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e805a0311eea683702ecc957a61057e2, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6f220ce002468367c756923f6fbc1fc1, ASSIGN}] 2024-12-05T13:17:16,623 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6f220ce002468367c756923f6fbc1fc1, ASSIGN 2024-12-05T13:17:16,625 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6f220ce002468367c756923f6fbc1fc1, ASSIGN; state=OFFLINE, location=88aaa70743e7,39293,1733404632706; forceNewPlan=false, retain=false 2024-12-05T13:17:16,627 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:16,629 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN; state=OFFLINE, location=88aaa70743e7,41495,1733404632848; forceNewPlan=false, retain=false 2024-12-05T13:17:16,630 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e805a0311eea683702ecc957a61057e2, ASSIGN 2024-12-05T13:17:16,632 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e805a0311eea683702ecc957a61057e2, ASSIGN; state=OFFLINE, location=88aaa70743e7,36055,1733404632807; forceNewPlan=false, retain=false 2024-12-05T13:17:16,634 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=abced629a2bc26273048aa2f4b6f8d9c, ASSIGN 2024-12-05T13:17:16,635 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=abced629a2bc26273048aa2f4b6f8d9c, ASSIGN; state=OFFLINE, location=88aaa70743e7,39293,1733404632706; forceNewPlan=false, retain=false 2024-12-05T13:17:16,637 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:16,638 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN; state=OFFLINE, location=88aaa70743e7,41495,1733404632848; forceNewPlan=false, retain=false 2024-12-05T13:17:16,640 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f4b86b8085620be60c3a91043e8f4d72, ASSIGN 2024-12-05T13:17:16,641 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f4b86b8085620be60c3a91043e8f4d72, ASSIGN; state=OFFLINE, location=88aaa70743e7,36055,1733404632807; forceNewPlan=false, retain=false 2024-12-05T13:17:16,643 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bb504ea56b81b22b5ca81b650f200ded, ASSIGN 2024-12-05T13:17:16,645 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bb504ea56b81b22b5ca81b650f200ded, ASSIGN; state=OFFLINE, location=88aaa70743e7,39293,1733404632706; forceNewPlan=false, retain=false 2024-12-05T13:17:16,646 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:16,648 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN; state=OFFLINE, location=88aaa70743e7,41495,1733404632848; forceNewPlan=false, retain=false 2024-12-05T13:17:16,649 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a362f9452e2fdf5e9192ce69e889c30d, ASSIGN 2024-12-05T13:17:16,651 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a362f9452e2fdf5e9192ce69e889c30d, ASSIGN; state=OFFLINE, location=88aaa70743e7,36055,1733404632807; forceNewPlan=false, retain=false 2024-12-05T13:17:16,652 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5d414b07d03c5766c12220a94662f4ff, ASSIGN 2024-12-05T13:17:16,654 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5d414b07d03c5766c12220a94662f4ff, ASSIGN; state=OFFLINE, location=88aaa70743e7,39293,1733404632706; forceNewPlan=false, retain=false 2024-12-05T13:17:16,659 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:16,661 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN; state=OFFLINE, location=88aaa70743e7,41495,1733404632848; forceNewPlan=false, retain=false 2024-12-05T13:17:16,662 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bd586ec5ee08f004669e3c48942b3638, ASSIGN 2024-12-05T13:17:16,664 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bd586ec5ee08f004669e3c48942b3638, ASSIGN; state=OFFLINE, location=88aaa70743e7,36055,1733404632807; forceNewPlan=false, retain=false 2024-12-05T13:17:16,665 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c086578586a3011e8fde7b2976cca84b, ASSIGN 2024-12-05T13:17:16,667 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c086578586a3011e8fde7b2976cca84b, ASSIGN; state=OFFLINE, location=88aaa70743e7,39293,1733404632706; forceNewPlan=false, retain=false 2024-12-05T13:17:16,668 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:16,670 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN; state=OFFLINE, location=88aaa70743e7,41495,1733404632848; forceNewPlan=false, retain=false 2024-12-05T13:17:16,671 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3ea448375edac1598b6cdff464d477bf, ASSIGN 2024-12-05T13:17:16,672 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3ea448375edac1598b6cdff464d477bf, ASSIGN; state=OFFLINE, location=88aaa70743e7,36055,1733404632807; forceNewPlan=false, retain=false 2024-12-05T13:17:16,674 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f29f7ad7d2427bd1681491cb1725214f, ASSIGN 2024-12-05T13:17:16,675 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f29f7ad7d2427bd1681491cb1725214f, ASSIGN; state=OFFLINE, location=88aaa70743e7,36055,1733404632807; forceNewPlan=false, retain=false 2024-12-05T13:17:16,676 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8d9d0bbe2316236e0d9afb8d72cd4099, ASSIGN 2024-12-05T13:17:16,677 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8d9d0bbe2316236e0d9afb8d72cd4099, ASSIGN; state=OFFLINE, location=88aaa70743e7,39293,1733404632706; forceNewPlan=false, retain=false 2024-12-05T13:17:16,678 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN 2024-12-05T13:17:16,679 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN; state=OFFLINE, location=88aaa70743e7,41495,1733404632848; forceNewPlan=false, retain=false 2024-12-05T13:17:16,681 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7bf9cb128da6b1afbc569bd4b25eba2d, ASSIGN 2024-12-05T13:17:16,682 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7bf9cb128da6b1afbc569bd4b25eba2d, ASSIGN; state=OFFLINE, location=88aaa70743e7,36055,1733404632807; forceNewPlan=false, retain=false 2024-12-05T13:17:16,683 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4debf2a1857c920c4d806684d7cb0807, ASSIGN 2024-12-05T13:17:16,685 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4debf2a1857c920c4d806684d7cb0807, ASSIGN; state=OFFLINE, location=88aaa70743e7,39293,1733404632706; forceNewPlan=false, retain=false 2024-12-05T13:17:16,686 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1218265d16f004d3f7900dc1adf62958, ASSIGN 2024-12-05T13:17:16,687 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1218265d16f004d3f7900dc1adf62958, ASSIGN; state=OFFLINE, location=88aaa70743e7,39293,1733404632706; forceNewPlan=false, retain=false 2024-12-05T13:17:16,689 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8e77e301280f5f9879631fbf6a099514, ASSIGN 2024-12-05T13:17:16,690 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8e77e301280f5f9879631fbf6a099514, ASSIGN; state=OFFLINE, location=88aaa70743e7,36055,1733404632807; forceNewPlan=false, retain=false 2024-12-05T13:17:16,691 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:16,692 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN; state=OFFLINE, location=88aaa70743e7,41495,1733404632848; forceNewPlan=false, retain=false 2024-12-05T13:17:16,694 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=943cec28fed0bc3f97c304818b208e3f, ASSIGN 2024-12-05T13:17:16,695 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=943cec28fed0bc3f97c304818b208e3f, ASSIGN; state=OFFLINE, location=88aaa70743e7,36055,1733404632807; forceNewPlan=false, retain=false 2024-12-05T13:17:16,697 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:16,700 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN; state=OFFLINE, location=88aaa70743e7,41495,1733404632848; forceNewPlan=false, retain=false 2024-12-05T13:17:16,703 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5c9639ed41e308974b4199260665db3c, ASSIGN 2024-12-05T13:17:16,704 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5c9639ed41e308974b4199260665db3c, ASSIGN; state=OFFLINE, location=88aaa70743e7,39293,1733404632706; forceNewPlan=false, retain=false 2024-12-05T13:17:16,778 INFO [88aaa70743e7:42815 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-12-05T13:17:16,779 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, regionLocation=88aaa70743e7,41495,1733404632848 2024-12-05T13:17:16,780 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, regionLocation=88aaa70743e7,41495,1733404632848 2024-12-05T13:17:16,781 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, regionLocation=88aaa70743e7,41495,1733404632848 2024-12-05T13:17:16,782 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, regionLocation=88aaa70743e7,41495,1733404632848 2024-12-05T13:17:16,783 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, regionLocation=88aaa70743e7,41495,1733404632848 2024-12-05T13:17:16,783 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, regionLocation=88aaa70743e7,41495,1733404632848 2024-12-05T13:17:16,784 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, regionLocation=88aaa70743e7,41495,1733404632848 2024-12-05T13:17:16,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN because future has completed 2024-12-05T13:17:16,785 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, regionLocation=88aaa70743e7,41495,1733404632848 2024-12-05T13:17:16,786 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=6f220ce002468367c756923f6fbc1fc1, regionState=OPENING, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:16,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure cabc686ba5a0c762c06d5a4d0024dd7b, server=88aaa70743e7,41495,1733404632848}] 2024-12-05T13:17:16,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN because future has completed 2024-12-05T13:17:16,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN because future has completed 2024-12-05T13:17:16,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN because future has completed 2024-12-05T13:17:16,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c03197a9f3300f90c023ccedb4f2112, server=88aaa70743e7,41495,1733404632848}] 2024-12-05T13:17:16,796 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN because future has completed 2024-12-05T13:17:16,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN because future has completed 2024-12-05T13:17:16,799 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN because future has completed 2024-12-05T13:17:16,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN because future has completed 2024-12-05T13:17:16,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b3cc71809f62cd16a817f5a172f68ff, server=88aaa70743e7,41495,1733404632848}] 2024-12-05T13:17:16,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6f220ce002468367c756923f6fbc1fc1, ASSIGN because future has completed 2024-12-05T13:17:16,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f220ce002468367c756923f6fbc1fc1, server=88aaa70743e7,39293,1733404632706}] 2024-12-05T13:17:16,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8705d72ae6f07bbf8fa8bfbf5a81d07, server=88aaa70743e7,41495,1733404632848}] 2024-12-05T13:17:16,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01ef88de67b81e433e4128dc0a819605, server=88aaa70743e7,41495,1733404632848}] 2024-12-05T13:17:16,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb2ca1f66d6c9b4513c79bf39e4e5bfe, server=88aaa70743e7,41495,1733404632848}] 2024-12-05T13:17:16,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,41495,1733404632848}] 2024-12-05T13:17:16,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb80ccb4b3541210c6fda8b4b988adb5, server=88aaa70743e7,41495,1733404632848}] 2024-12-05T13:17:16,852 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPENING, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:16,853 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=bb504ea56b81b22b5ca81b650f200ded, regionState=OPENING, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:16,853 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=5d414b07d03c5766c12220a94662f4ff, regionState=OPENING, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:16,854 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=c086578586a3011e8fde7b2976cca84b, regionState=OPENING, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:16,855 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPENING, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:16,856 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=4debf2a1857c920c4d806684d7cb0807, regionState=OPENING, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:16,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=abced629a2bc26273048aa2f4b6f8d9c, ASSIGN because future has completed 2024-12-05T13:17:16,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure abced629a2bc26273048aa2f4b6f8d9c, server=88aaa70743e7,39293,1733404632706}] 2024-12-05T13:17:16,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bb504ea56b81b22b5ca81b650f200ded, ASSIGN because future has completed 2024-12-05T13:17:16,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5d414b07d03c5766c12220a94662f4ff, ASSIGN because future has completed 2024-12-05T13:17:16,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c086578586a3011e8fde7b2976cca84b, ASSIGN because future has completed 2024-12-05T13:17:16,866 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8d9d0bbe2316236e0d9afb8d72cd4099, ASSIGN because future has completed 2024-12-05T13:17:16,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d9d0bbe2316236e0d9afb8d72cd4099, server=88aaa70743e7,39293,1733404632706}] 2024-12-05T13:17:16,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4debf2a1857c920c4d806684d7cb0807, ASSIGN because future has completed 2024-12-05T13:17:16,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4debf2a1857c920c4d806684d7cb0807, server=88aaa70743e7,39293,1733404632706}] 2024-12-05T13:17:16,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure c086578586a3011e8fde7b2976cca84b, server=88aaa70743e7,39293,1733404632706}] 2024-12-05T13:17:16,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d414b07d03c5766c12220a94662f4ff, server=88aaa70743e7,39293,1733404632706}] 2024-12-05T13:17:16,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb504ea56b81b22b5ca81b650f200ded, server=88aaa70743e7,39293,1733404632706}] 2024-12-05T13:17:16,897 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1218265d16f004d3f7900dc1adf62958, regionState=OPENING, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:16,898 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5c9639ed41e308974b4199260665db3c, regionState=OPENING, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:16,899 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=e805a0311eea683702ecc957a61057e2, regionState=OPENING, regionLocation=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:16,900 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=f4b86b8085620be60c3a91043e8f4d72, regionState=OPENING, regionLocation=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:16,901 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPENING, regionLocation=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:16,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1218265d16f004d3f7900dc1adf62958, ASSIGN because future has completed 2024-12-05T13:17:16,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1218265d16f004d3f7900dc1adf62958, server=88aaa70743e7,39293,1733404632706}] 2024-12-05T13:17:16,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5c9639ed41e308974b4199260665db3c, ASSIGN because future has completed 2024-12-05T13:17:16,908 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e805a0311eea683702ecc957a61057e2, ASSIGN because future has completed 2024-12-05T13:17:16,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure e805a0311eea683702ecc957a61057e2, server=88aaa70743e7,36055,1733404632807}] 2024-12-05T13:17:16,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f4b86b8085620be60c3a91043e8f4d72, ASSIGN because future has completed 2024-12-05T13:17:16,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a362f9452e2fdf5e9192ce69e889c30d, ASSIGN because future has completed 2024-12-05T13:17:16,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure a362f9452e2fdf5e9192ce69e889c30d, server=88aaa70743e7,36055,1733404632807}] 2024-12-05T13:17:16,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure f4b86b8085620be60c3a91043e8f4d72, server=88aaa70743e7,36055,1733404632807}] 2024-12-05T13:17:16,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5c9639ed41e308974b4199260665db3c, server=88aaa70743e7,39293,1733404632706}] 2024-12-05T13:17:16,929 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=bd586ec5ee08f004669e3c48942b3638, regionState=OPENING, regionLocation=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:16,929 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=3ea448375edac1598b6cdff464d477bf, regionState=OPENING, regionLocation=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:16,930 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=f29f7ad7d2427bd1681491cb1725214f, regionState=OPENING, regionLocation=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:16,931 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPENING, regionLocation=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:16,931 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=8e77e301280f5f9879631fbf6a099514, regionState=OPENING, regionLocation=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:16,932 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=943cec28fed0bc3f97c304818b208e3f, regionState=OPENING, regionLocation=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:16,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bd586ec5ee08f004669e3c48942b3638, ASSIGN because future has completed 2024-12-05T13:17:16,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd586ec5ee08f004669e3c48942b3638, server=88aaa70743e7,36055,1733404632807}] 2024-12-05T13:17:16,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3ea448375edac1598b6cdff464d477bf, ASSIGN because future has completed 2024-12-05T13:17:16,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f29f7ad7d2427bd1681491cb1725214f, ASSIGN because future has completed 2024-12-05T13:17:16,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7bf9cb128da6b1afbc569bd4b25eba2d, ASSIGN because future has completed 2024-12-05T13:17:16,939 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7bf9cb128da6b1afbc569bd4b25eba2d, server=88aaa70743e7,36055,1733404632807}] 2024-12-05T13:17:16,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8e77e301280f5f9879631fbf6a099514, ASSIGN because future has completed 2024-12-05T13:17:16,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=943cec28fed0bc3f97c304818b208e3f, ASSIGN because future has completed 2024-12-05T13:17:16,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 943cec28fed0bc3f97c304818b208e3f, server=88aaa70743e7,36055,1733404632807}] 2024-12-05T13:17:16,948 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe. 2024-12-05T13:17:16,948 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => eb2ca1f66d6c9b4513c79bf39e4e5bfe, NAME => 'test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-12-05T13:17:16,949 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:16,949 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,949 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:16,949 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:16,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e77e301280f5f9879631fbf6a099514, server=88aaa70743e7,36055,1733404632807}] 2024-12-05T13:17:16,952 INFO [StoreOpener-eb2ca1f66d6c9b4513c79bf39e4e5bfe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:16,955 INFO [StoreOpener-eb2ca1f66d6c9b4513c79bf39e4e5bfe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb2ca1f66d6c9b4513c79bf39e4e5bfe columnFamilyName family 2024-12-05T13:17:16,955 DEBUG [StoreOpener-eb2ca1f66d6c9b4513c79bf39e4e5bfe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:16,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure f29f7ad7d2427bd1681491cb1725214f, server=88aaa70743e7,36055,1733404632807}] 2024-12-05T13:17:16,956 INFO [StoreOpener-eb2ca1f66d6c9b4513c79bf39e4e5bfe-1 {}] regionserver.HStore(327): Store=eb2ca1f66d6c9b4513c79bf39e4e5bfe/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:16,956 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:16,958 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:16,959 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:16,960 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:16,960 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:16,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3ea448375edac1598b6cdff464d477bf, server=88aaa70743e7,36055,1733404632807}] 2024-12-05T13:17:16,963 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:16,969 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T13:17:16,972 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49345, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T13:17:16,978 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,qqq,1733404635844.5d414b07d03c5766c12220a94662f4ff. 2024-12-05T13:17:16,978 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 5d414b07d03c5766c12220a94662f4ff, NAME => 'test,qqq,1733404635844.5d414b07d03c5766c12220a94662f4ff.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-12-05T13:17:16,979 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/eb2ca1f66d6c9b4513c79bf39e4e5bfe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:16,979 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5d414b07d03c5766c12220a94662f4ff 2024-12-05T13:17:16,979 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,qqq,1733404635844.5d414b07d03c5766c12220a94662f4ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,979 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 5d414b07d03c5766c12220a94662f4ff 2024-12-05T13:17:16,979 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 5d414b07d03c5766c12220a94662f4ff 2024-12-05T13:17:16,980 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened eb2ca1f66d6c9b4513c79bf39e4e5bfe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66214266, jitterRate=-0.013330549001693726}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:16,980 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:16,981 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for eb2ca1f66d6c9b4513c79bf39e4e5bfe: Running coprocessor pre-open hook at 1733404636949Writing region info on filesystem at 1733404636950 (+1 ms)Initializing all the Stores at 1733404636951 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404636951Cleaning up temporary data from old regions at 1733404636960 (+9 ms)Running coprocessor post-open hooks at 1733404636980 (+20 ms)Region opened successfully at 1733404636981 (+1 ms) 2024-12-05T13:17:16,981 INFO [StoreOpener-5d414b07d03c5766c12220a94662f4ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5d414b07d03c5766c12220a94662f4ff 2024-12-05T13:17:16,983 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe., pid=37, masterSystemTime=1733404636942 2024-12-05T13:17:16,984 INFO [StoreOpener-5d414b07d03c5766c12220a94662f4ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5d414b07d03c5766c12220a94662f4ff columnFamilyName family 2024-12-05T13:17:16,984 DEBUG [StoreOpener-5d414b07d03c5766c12220a94662f4ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:16,985 INFO [StoreOpener-5d414b07d03c5766c12220a94662f4ff-1 {}] regionserver.HStore(327): Store=5d414b07d03c5766c12220a94662f4ff/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:16,986 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 5d414b07d03c5766c12220a94662f4ff 2024-12-05T13:17:16,986 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe. 2024-12-05T13:17:16,987 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe. 2024-12-05T13:17:16,987 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07. 2024-12-05T13:17:16,987 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => f8705d72ae6f07bbf8fa8bfbf5a81d07, NAME => 'test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-12-05T13:17:16,988 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:16,988 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:16,988 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:16,988 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:16,988 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,41495,1733404632848 2024-12-05T13:17:16,990 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/5d414b07d03c5766c12220a94662f4ff 2024-12-05T13:17:16,991 INFO [StoreOpener-f8705d72ae6f07bbf8fa8bfbf5a81d07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:16,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb2ca1f66d6c9b4513c79bf39e4e5bfe, server=88aaa70743e7,41495,1733404632848 because future has completed 2024-12-05T13:17:16,994 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=88aaa70743e7,41495,1733404632848, table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T13:17:16,995 INFO [StoreOpener-f8705d72ae6f07bbf8fa8bfbf5a81d07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8705d72ae6f07bbf8fa8bfbf5a81d07 columnFamilyName family 2024-12-05T13:17:16,995 DEBUG [StoreOpener-f8705d72ae6f07bbf8fa8bfbf5a81d07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:16,996 INFO [StoreOpener-f8705d72ae6f07bbf8fa8bfbf5a81d07-1 {}] regionserver.HStore(327): Store=f8705d72ae6f07bbf8fa8bfbf5a81d07/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:16,997 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/5d414b07d03c5766c12220a94662f4ff 2024-12-05T13:17:16,997 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:16,998 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:16,998 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 5d414b07d03c5766c12220a94662f4ff 2024-12-05T13:17:16,998 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 5d414b07d03c5766c12220a94662f4ff 2024-12-05T13:17:16,999 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:17,000 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:17,000 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:17,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=17 2024-12-05T13:17:17,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure eb2ca1f66d6c9b4513c79bf39e4e5bfe, server=88aaa70743e7,41495,1733404632848 in 165 msec 2024-12-05T13:17:17,002 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 5d414b07d03c5766c12220a94662f4ff 2024-12-05T13:17:17,003 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:17,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN in 389 msec 2024-12-05T13:17:17,006 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/5d414b07d03c5766c12220a94662f4ff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,007 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 5d414b07d03c5766c12220a94662f4ff; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64509132, jitterRate=-0.038739025592803955}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,007 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5d414b07d03c5766c12220a94662f4ff 2024-12-05T13:17:17,007 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 5d414b07d03c5766c12220a94662f4ff: Running coprocessor pre-open hook at 1733404636979Writing region info on filesystem at 1733404636979Initializing all the Stores at 1733404636981 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404636981Cleaning up temporary data from old regions at 1733404636998 (+17 ms)Running coprocessor post-open hooks at 1733404637007 (+9 ms)Region opened successfully at 1733404637007 2024-12-05T13:17:17,008 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1733404635844.5d414b07d03c5766c12220a94662f4ff., pid=44, masterSystemTime=1733404636969 2024-12-05T13:17:17,012 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/f8705d72ae6f07bbf8fa8bfbf5a81d07/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,013 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1733404635844.5d414b07d03c5766c12220a94662f4ff. 2024-12-05T13:17:17,013 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened f8705d72ae6f07bbf8fa8bfbf5a81d07; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59823069, jitterRate=-0.10856680572032928}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,013 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,qqq,1733404635844.5d414b07d03c5766c12220a94662f4ff. 2024-12-05T13:17:17,013 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:17,013 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,nnn,1733404635844.c086578586a3011e8fde7b2976cca84b. 2024-12-05T13:17:17,013 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for f8705d72ae6f07bbf8fa8bfbf5a81d07: Running coprocessor pre-open hook at 1733404636988Writing region info on filesystem at 1733404636988Initializing all the Stores at 1733404636991 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404636991Cleaning up temporary data from old regions at 1733404637000 (+9 ms)Running coprocessor post-open hooks at 1733404637013 (+13 ms)Region opened successfully at 1733404637013 2024-12-05T13:17:17,013 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:17,013 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => c086578586a3011e8fde7b2976cca84b, NAME => 'test,nnn,1733404635844.c086578586a3011e8fde7b2976cca84b.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-12-05T13:17:17,014 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c086578586a3011e8fde7b2976cca84b 2024-12-05T13:17:17,014 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,nnn,1733404635844.c086578586a3011e8fde7b2976cca84b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,014 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for c086578586a3011e8fde7b2976cca84b 2024-12-05T13:17:17,014 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for c086578586a3011e8fde7b2976cca84b 2024-12-05T13:17:17,016 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07., pid=35, masterSystemTime=1733404636942 2024-12-05T13:17:17,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5d414b07d03c5766c12220a94662f4ff, server=88aaa70743e7,39293,1733404632706 because future has completed 2024-12-05T13:17:17,021 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07. 2024-12-05T13:17:17,021 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07. 2024-12-05T13:17:17,021 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,iii,1733404635844.01ef88de67b81e433e4128dc0a819605. 2024-12-05T13:17:17,022 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 01ef88de67b81e433e4128dc0a819605, NAME => 'test,iii,1733404635844.01ef88de67b81e433e4128dc0a819605.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-12-05T13:17:17,022 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 01ef88de67b81e433e4128dc0a819605 2024-12-05T13:17:17,022 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,iii,1733404635844.01ef88de67b81e433e4128dc0a819605.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,022 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 01ef88de67b81e433e4128dc0a819605 2024-12-05T13:17:17,023 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 01ef88de67b81e433e4128dc0a819605 2024-12-05T13:17:17,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=21 2024-12-05T13:17:17,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 5d414b07d03c5766c12220a94662f4ff, server=88aaa70743e7,39293,1733404632706 in 134 msec 2024-12-05T13:17:17,027 INFO [StoreOpener-c086578586a3011e8fde7b2976cca84b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c086578586a3011e8fde7b2976cca84b 2024-12-05T13:17:17,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5d414b07d03c5766c12220a94662f4ff, ASSIGN in 414 msec 2024-12-05T13:17:17,032 INFO [StoreOpener-c086578586a3011e8fde7b2976cca84b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c086578586a3011e8fde7b2976cca84b columnFamilyName family 2024-12-05T13:17:17,032 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,41495,1733404632848 2024-12-05T13:17:17,032 DEBUG [StoreOpener-c086578586a3011e8fde7b2976cca84b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,033 INFO [StoreOpener-01ef88de67b81e433e4128dc0a819605-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 01ef88de67b81e433e4128dc0a819605 2024-12-05T13:17:17,035 INFO [StoreOpener-c086578586a3011e8fde7b2976cca84b-1 {}] regionserver.HStore(327): Store=c086578586a3011e8fde7b2976cca84b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,035 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for c086578586a3011e8fde7b2976cca84b 2024-12-05T13:17:17,035 INFO [StoreOpener-01ef88de67b81e433e4128dc0a819605-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01ef88de67b81e433e4128dc0a819605 columnFamilyName family 2024-12-05T13:17:17,035 DEBUG [StoreOpener-01ef88de67b81e433e4128dc0a819605-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,036 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/c086578586a3011e8fde7b2976cca84b 2024-12-05T13:17:17,036 INFO [StoreOpener-01ef88de67b81e433e4128dc0a819605-1 {}] regionserver.HStore(327): Store=01ef88de67b81e433e4128dc0a819605/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8705d72ae6f07bbf8fa8bfbf5a81d07, server=88aaa70743e7,41495,1733404632848 because future has completed 2024-12-05T13:17:17,036 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 01ef88de67b81e433e4128dc0a819605 2024-12-05T13:17:17,037 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/c086578586a3011e8fde7b2976cca84b 2024-12-05T13:17:17,037 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for c086578586a3011e8fde7b2976cca84b 2024-12-05T13:17:17,037 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/01ef88de67b81e433e4128dc0a819605 2024-12-05T13:17:17,038 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for c086578586a3011e8fde7b2976cca84b 2024-12-05T13:17:17,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T13:17:17,039 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/01ef88de67b81e433e4128dc0a819605 2024-12-05T13:17:17,040 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 01ef88de67b81e433e4128dc0a819605 2024-12-05T13:17:17,040 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 01ef88de67b81e433e4128dc0a819605 2024-12-05T13:17:17,041 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for c086578586a3011e8fde7b2976cca84b 2024-12-05T13:17:17,043 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 01ef88de67b81e433e4128dc0a819605 2024-12-05T13:17:17,045 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/c086578586a3011e8fde7b2976cca84b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=8 2024-12-05T13:17:17,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure f8705d72ae6f07bbf8fa8bfbf5a81d07, server=88aaa70743e7,41495,1733404632848 in 221 msec 2024-12-05T13:17:17,046 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened c086578586a3011e8fde7b2976cca84b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63119330, jitterRate=-0.059448689222335815}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,046 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c086578586a3011e8fde7b2976cca84b 2024-12-05T13:17:17,047 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for c086578586a3011e8fde7b2976cca84b: Running coprocessor pre-open hook at 1733404637015Writing region info on filesystem at 1733404637015Initializing all the Stores at 1733404637017 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637017Cleaning up temporary data from old regions at 1733404637038 (+21 ms)Running coprocessor post-open hooks at 1733404637047 (+9 ms)Region opened successfully at 1733404637047 2024-12-05T13:17:17,048 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/01ef88de67b81e433e4128dc0a819605/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,048 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1733404635844.c086578586a3011e8fde7b2976cca84b., pid=43, masterSystemTime=1733404636969 2024-12-05T13:17:17,049 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN in 434 msec 2024-12-05T13:17:17,049 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 01ef88de67b81e433e4128dc0a819605; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71775298, jitterRate=0.0695352852344513}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,050 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 01ef88de67b81e433e4128dc0a819605 2024-12-05T13:17:17,050 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 01ef88de67b81e433e4128dc0a819605: Running coprocessor pre-open hook at 1733404637023Writing region info on filesystem at 1733404637023Initializing all the Stores at 1733404637028 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637028Cleaning up temporary data from old regions at 1733404637040 (+12 ms)Running coprocessor post-open hooks at 1733404637050 (+10 ms)Region opened successfully at 1733404637050 2024-12-05T13:17:17,051 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1733404635844.01ef88de67b81e433e4128dc0a819605., pid=36, masterSystemTime=1733404636942 2024-12-05T13:17:17,052 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1733404635844.c086578586a3011e8fde7b2976cca84b. 2024-12-05T13:17:17,052 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,nnn,1733404635844.c086578586a3011e8fde7b2976cca84b. 2024-12-05T13:17:17,053 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,ttt,1733404635844.bb504ea56b81b22b5ca81b650f200ded. 2024-12-05T13:17:17,053 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:17,053 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => bb504ea56b81b22b5ca81b650f200ded, NAME => 'test,ttt,1733404635844.bb504ea56b81b22b5ca81b650f200ded.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-12-05T13:17:17,053 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bb504ea56b81b22b5ca81b650f200ded 2024-12-05T13:17:17,053 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,ttt,1733404635844.bb504ea56b81b22b5ca81b650f200ded.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,054 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for bb504ea56b81b22b5ca81b650f200ded 2024-12-05T13:17:17,054 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for bb504ea56b81b22b5ca81b650f200ded 2024-12-05T13:17:17,056 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1733404635844.01ef88de67b81e433e4128dc0a819605. 2024-12-05T13:17:17,056 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,iii,1733404635844.01ef88de67b81e433e4128dc0a819605. 2024-12-05T13:17:17,056 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,bbb,1733404635844.8b3cc71809f62cd16a817f5a172f68ff. 2024-12-05T13:17:17,057 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 8b3cc71809f62cd16a817f5a172f68ff, NAME => 'test,bbb,1733404635844.8b3cc71809f62cd16a817f5a172f68ff.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-12-05T13:17:17,057 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8b3cc71809f62cd16a817f5a172f68ff 2024-12-05T13:17:17,057 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,bbb,1733404635844.8b3cc71809f62cd16a817f5a172f68ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,058 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 8b3cc71809f62cd16a817f5a172f68ff 2024-12-05T13:17:17,058 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 8b3cc71809f62cd16a817f5a172f68ff 2024-12-05T13:17:17,059 INFO [StoreOpener-bb504ea56b81b22b5ca81b650f200ded-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bb504ea56b81b22b5ca81b650f200ded 2024-12-05T13:17:17,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure c086578586a3011e8fde7b2976cca84b, server=88aaa70743e7,39293,1733404632706 because future has completed 2024-12-05T13:17:17,061 INFO [StoreOpener-8b3cc71809f62cd16a817f5a172f68ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8b3cc71809f62cd16a817f5a172f68ff 2024-12-05T13:17:17,062 INFO [StoreOpener-bb504ea56b81b22b5ca81b650f200ded-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb504ea56b81b22b5ca81b650f200ded columnFamilyName family 2024-12-05T13:17:17,062 DEBUG [StoreOpener-bb504ea56b81b22b5ca81b650f200ded-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,063 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T13:17:17,064 INFO [StoreOpener-bb504ea56b81b22b5ca81b650f200ded-1 {}] regionserver.HStore(327): Store=bb504ea56b81b22b5ca81b650f200ded/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,064 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for bb504ea56b81b22b5ca81b650f200ded 2024-12-05T13:17:17,066 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/bb504ea56b81b22b5ca81b650f200ded 2024-12-05T13:17:17,066 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46337, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T13:17:17,066 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/bb504ea56b81b22b5ca81b650f200ded 2024-12-05T13:17:17,067 INFO [StoreOpener-8b3cc71809f62cd16a817f5a172f68ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b3cc71809f62cd16a817f5a172f68ff columnFamilyName family 2024-12-05T13:17:17,067 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for bb504ea56b81b22b5ca81b650f200ded 2024-12-05T13:17:17,067 DEBUG [StoreOpener-8b3cc71809f62cd16a817f5a172f68ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,067 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for bb504ea56b81b22b5ca81b650f200ded 2024-12-05T13:17:17,068 INFO [StoreOpener-8b3cc71809f62cd16a817f5a172f68ff-1 {}] regionserver.HStore(327): Store=8b3cc71809f62cd16a817f5a172f68ff/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,069 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 8b3cc71809f62cd16a817f5a172f68ff 2024-12-05T13:17:17,073 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/8b3cc71809f62cd16a817f5a172f68ff 2024-12-05T13:17:17,080 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/8b3cc71809f62cd16a817f5a172f68ff 2024-12-05T13:17:17,081 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 8b3cc71809f62cd16a817f5a172f68ff 2024-12-05T13:17:17,082 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 8b3cc71809f62cd16a817f5a172f68ff 2024-12-05T13:17:17,084 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for bb504ea56b81b22b5ca81b650f200ded 2024-12-05T13:17:17,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=18 2024-12-05T13:17:17,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure c086578586a3011e8fde7b2976cca84b, server=88aaa70743e7,39293,1733404632706 in 181 msec 2024-12-05T13:17:17,093 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 8b3cc71809f62cd16a817f5a172f68ff 2024-12-05T13:17:17,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c086578586a3011e8fde7b2976cca84b, ASSIGN in 480 msec 2024-12-05T13:17:17,103 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=01ef88de67b81e433e4128dc0a819605, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,41495,1733404632848 2024-12-05T13:17:17,106 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,hhh,1733404635844.7bf9cb128da6b1afbc569bd4b25eba2d. 2024-12-05T13:17:17,106 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 7bf9cb128da6b1afbc569bd4b25eba2d, NAME => 'test,hhh,1733404635844.7bf9cb128da6b1afbc569bd4b25eba2d.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-12-05T13:17:17,107 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7bf9cb128da6b1afbc569bd4b25eba2d 2024-12-05T13:17:17,107 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,hhh,1733404635844.7bf9cb128da6b1afbc569bd4b25eba2d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,107 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 7bf9cb128da6b1afbc569bd4b25eba2d 2024-12-05T13:17:17,107 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 7bf9cb128da6b1afbc569bd4b25eba2d 2024-12-05T13:17:17,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01ef88de67b81e433e4128dc0a819605, server=88aaa70743e7,41495,1733404632848 because future has completed 2024-12-05T13:17:17,112 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/bb504ea56b81b22b5ca81b650f200ded/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,113 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/8b3cc71809f62cd16a817f5a172f68ff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,114 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 8b3cc71809f62cd16a817f5a172f68ff; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74250996, jitterRate=0.10642606019973755}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,114 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8b3cc71809f62cd16a817f5a172f68ff 2024-12-05T13:17:17,114 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 8b3cc71809f62cd16a817f5a172f68ff: Running coprocessor pre-open hook at 1733404637058Writing region info on filesystem at 1733404637058Initializing all the Stores at 1733404637060 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637060Cleaning up temporary data from old regions at 1733404637082 (+22 ms)Running coprocessor post-open hooks at 1733404637114 (+32 ms)Region opened successfully at 1733404637114 2024-12-05T13:17:17,116 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1733404635844.8b3cc71809f62cd16a817f5a172f68ff., pid=33, masterSystemTime=1733404636942 2024-12-05T13:17:17,116 INFO [StoreOpener-7bf9cb128da6b1afbc569bd4b25eba2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7bf9cb128da6b1afbc569bd4b25eba2d 2024-12-05T13:17:17,120 INFO [StoreOpener-7bf9cb128da6b1afbc569bd4b25eba2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7bf9cb128da6b1afbc569bd4b25eba2d columnFamilyName family 2024-12-05T13:17:17,120 DEBUG [StoreOpener-7bf9cb128da6b1afbc569bd4b25eba2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,120 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1733404635844.8b3cc71809f62cd16a817f5a172f68ff. 2024-12-05T13:17:17,120 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,bbb,1733404635844.8b3cc71809f62cd16a817f5a172f68ff. 2024-12-05T13:17:17,120 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3. 2024-12-05T13:17:17,121 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 97426aa8ec496bfcaaf589a5cbe6f1b3, NAME => 'test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-12-05T13:17:17,121 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:17,121 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,121 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:17,122 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:17,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=13 2024-12-05T13:17:17,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 01ef88de67b81e433e4128dc0a819605, server=88aaa70743e7,41495,1733404632848 in 285 msec 2024-12-05T13:17:17,123 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened bb504ea56b81b22b5ca81b650f200ded; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60039330, jitterRate=-0.1053442656993866}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,123 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bb504ea56b81b22b5ca81b650f200ded 2024-12-05T13:17:17,123 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for bb504ea56b81b22b5ca81b650f200ded: Running coprocessor pre-open hook at 1733404637054Writing region info on filesystem at 1733404637054Initializing all the Stores at 1733404637057 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637057Cleaning up temporary data from old regions at 1733404637067 (+10 ms)Running coprocessor post-open hooks at 1733404637123 (+56 ms)Region opened successfully at 1733404637123 2024-12-05T13:17:17,124 INFO [StoreOpener-7bf9cb128da6b1afbc569bd4b25eba2d-1 {}] regionserver.HStore(327): Store=7bf9cb128da6b1afbc569bd4b25eba2d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,125 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 7bf9cb128da6b1afbc569bd4b25eba2d 2024-12-05T13:17:17,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN in 510 msec 2024-12-05T13:17:17,126 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/7bf9cb128da6b1afbc569bd4b25eba2d 2024-12-05T13:17:17,127 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1733404635844.bb504ea56b81b22b5ca81b650f200ded., pid=45, masterSystemTime=1733404636969 2024-12-05T13:17:17,128 INFO [StoreOpener-97426aa8ec496bfcaaf589a5cbe6f1b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:17,130 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,41495,1733404632848 2024-12-05T13:17:17,130 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1733404635844.bb504ea56b81b22b5ca81b650f200ded. 2024-12-05T13:17:17,131 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,ttt,1733404635844.bb504ea56b81b22b5ca81b650f200ded. 2024-12-05T13:17:17,131 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,www,1733404635844.abced629a2bc26273048aa2f4b6f8d9c. 2024-12-05T13:17:17,131 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => abced629a2bc26273048aa2f4b6f8d9c, NAME => 'test,www,1733404635844.abced629a2bc26273048aa2f4b6f8d9c.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-12-05T13:17:17,132 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test abced629a2bc26273048aa2f4b6f8d9c 2024-12-05T13:17:17,132 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,www,1733404635844.abced629a2bc26273048aa2f4b6f8d9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,132 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for abced629a2bc26273048aa2f4b6f8d9c 2024-12-05T13:17:17,132 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for abced629a2bc26273048aa2f4b6f8d9c 2024-12-05T13:17:17,133 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/7bf9cb128da6b1afbc569bd4b25eba2d 2024-12-05T13:17:17,134 INFO [StoreOpener-97426aa8ec496bfcaaf589a5cbe6f1b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97426aa8ec496bfcaaf589a5cbe6f1b3 columnFamilyName family 2024-12-05T13:17:17,134 DEBUG [StoreOpener-97426aa8ec496bfcaaf589a5cbe6f1b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,135 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 7bf9cb128da6b1afbc569bd4b25eba2d 2024-12-05T13:17:17,135 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 7bf9cb128da6b1afbc569bd4b25eba2d 2024-12-05T13:17:17,135 INFO [StoreOpener-97426aa8ec496bfcaaf589a5cbe6f1b3-1 {}] regionserver.HStore(327): Store=97426aa8ec496bfcaaf589a5cbe6f1b3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,135 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:17,136 INFO [StoreOpener-abced629a2bc26273048aa2f4b6f8d9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region abced629a2bc26273048aa2f4b6f8d9c 2024-12-05T13:17:17,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b3cc71809f62cd16a817f5a172f68ff, server=88aaa70743e7,41495,1733404632848 because future has completed 2024-12-05T13:17:17,137 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:17,139 INFO [StoreOpener-abced629a2bc26273048aa2f4b6f8d9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abced629a2bc26273048aa2f4b6f8d9c columnFamilyName family 2024-12-05T13:17:17,139 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:17,139 DEBUG [StoreOpener-abced629a2bc26273048aa2f4b6f8d9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,139 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:17,141 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:17,141 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:17,141 INFO [StoreOpener-abced629a2bc26273048aa2f4b6f8d9c-1 {}] regionserver.HStore(327): Store=abced629a2bc26273048aa2f4b6f8d9c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,141 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 7bf9cb128da6b1afbc569bd4b25eba2d 2024-12-05T13:17:17,141 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for abced629a2bc26273048aa2f4b6f8d9c 2024-12-05T13:17:17,145 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/abced629a2bc26273048aa2f4b6f8d9c 2024-12-05T13:17:17,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb504ea56b81b22b5ca81b650f200ded, server=88aaa70743e7,39293,1733404632706 because future has completed 2024-12-05T13:17:17,146 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/abced629a2bc26273048aa2f4b6f8d9c 2024-12-05T13:17:17,146 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:17,146 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for abced629a2bc26273048aa2f4b6f8d9c 2024-12-05T13:17:17,147 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for abced629a2bc26273048aa2f4b6f8d9c 2024-12-05T13:17:17,148 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/7bf9cb128da6b1afbc569bd4b25eba2d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,149 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 7bf9cb128da6b1afbc569bd4b25eba2d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63996727, jitterRate=-0.04637445509433746}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,149 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7bf9cb128da6b1afbc569bd4b25eba2d 2024-12-05T13:17:17,149 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 7bf9cb128da6b1afbc569bd4b25eba2d: Running coprocessor pre-open hook at 1733404637107Writing region info on filesystem at 1733404637107Initializing all the Stores at 1733404637109 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637109Cleaning up temporary data from old regions at 1733404637135 (+26 ms)Running coprocessor post-open hooks at 1733404637149 (+14 ms)Region opened successfully at 1733404637149 2024-12-05T13:17:17,151 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for abced629a2bc26273048aa2f4b6f8d9c 2024-12-05T13:17:17,151 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1733404635844.7bf9cb128da6b1afbc569bd4b25eba2d., pid=52, masterSystemTime=1733404637063 2024-12-05T13:17:17,153 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/97426aa8ec496bfcaaf589a5cbe6f1b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,154 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 97426aa8ec496bfcaaf589a5cbe6f1b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63900835, jitterRate=-0.0478033572435379}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,155 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:17,155 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 97426aa8ec496bfcaaf589a5cbe6f1b3: Running coprocessor pre-open hook at 1733404637122Writing region info on filesystem at 1733404637122Initializing all the Stores at 1733404637124 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637124Cleaning up temporary data from old regions at 1733404637141 (+17 ms)Running coprocessor post-open hooks at 1733404637155 (+14 ms)Region opened successfully at 1733404637155 2024-12-05T13:17:17,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=6 2024-12-05T13:17:17,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 8b3cc71809f62cd16a817f5a172f68ff, server=88aaa70743e7,41495,1733404632848 in 338 msec 2024-12-05T13:17:17,156 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1733404635844.7bf9cb128da6b1afbc569bd4b25eba2d. 2024-12-05T13:17:17,156 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,hhh,1733404635844.7bf9cb128da6b1afbc569bd4b25eba2d. 2024-12-05T13:17:17,156 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,ccc,1733404635844.943cec28fed0bc3f97c304818b208e3f. 2024-12-05T13:17:17,156 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3., pid=38, masterSystemTime=1733404636942 2024-12-05T13:17:17,156 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 943cec28fed0bc3f97c304818b208e3f, NAME => 'test,ccc,1733404635844.943cec28fed0bc3f97c304818b208e3f.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-12-05T13:17:17,157 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 943cec28fed0bc3f97c304818b208e3f 2024-12-05T13:17:17,157 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,ccc,1733404635844.943cec28fed0bc3f97c304818b208e3f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,157 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 943cec28fed0bc3f97c304818b208e3f 2024-12-05T13:17:17,157 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 943cec28fed0bc3f97c304818b208e3f 2024-12-05T13:17:17,160 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:17,160 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3. 2024-12-05T13:17:17,160 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3. 2024-12-05T13:17:17,160 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/abced629a2bc26273048aa2f4b6f8d9c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,160 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b. 2024-12-05T13:17:17,161 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => cabc686ba5a0c762c06d5a4d0024dd7b, NAME => 'test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-12-05T13:17:17,161 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:17,161 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,161 INFO [StoreOpener-943cec28fed0bc3f97c304818b208e3f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 943cec28fed0bc3f97c304818b208e3f 2024-12-05T13:17:17,162 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:17,162 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:17,166 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,41495,1733404632848 2024-12-05T13:17:17,167 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened abced629a2bc26273048aa2f4b6f8d9c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66167923, jitterRate=-0.014021113514900208}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,167 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for abced629a2bc26273048aa2f4b6f8d9c 2024-12-05T13:17:17,168 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for abced629a2bc26273048aa2f4b6f8d9c: Running coprocessor pre-open hook at 1733404637132Writing region info on filesystem at 1733404637132Initializing all the Stores at 1733404637135 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637135Cleaning up temporary data from old regions at 1733404637147 (+12 ms)Running coprocessor post-open hooks at 1733404637167 (+20 ms)Region opened successfully at 1733404637168 (+1 ms) 2024-12-05T13:17:17,168 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7bf9cb128da6b1afbc569bd4b25eba2d, server=88aaa70743e7,36055,1733404632807 because future has completed 2024-12-05T13:17:17,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,41495,1733404632848 because future has completed 2024-12-05T13:17:17,171 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1733404635844.abced629a2bc26273048aa2f4b6f8d9c., pid=40, masterSystemTime=1733404636969 2024-12-05T13:17:17,174 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1733404635844.abced629a2bc26273048aa2f4b6f8d9c. 2024-12-05T13:17:17,174 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,www,1733404635844.abced629a2bc26273048aa2f4b6f8d9c. 2024-12-05T13:17:17,174 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,,1733404635844.5c9639ed41e308974b4199260665db3c. 2024-12-05T13:17:17,174 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 5c9639ed41e308974b4199260665db3c, NAME => 'test,,1733404635844.5c9639ed41e308974b4199260665db3c.', STARTKEY => '', ENDKEY => 'bbb'} 2024-12-05T13:17:17,175 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5c9639ed41e308974b4199260665db3c 2024-12-05T13:17:17,175 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,,1733404635844.5c9639ed41e308974b4199260665db3c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,175 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 5c9639ed41e308974b4199260665db3c 2024-12-05T13:17:17,175 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 5c9639ed41e308974b4199260665db3c 2024-12-05T13:17:17,176 INFO [StoreOpener-943cec28fed0bc3f97c304818b208e3f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 943cec28fed0bc3f97c304818b208e3f columnFamilyName family 2024-12-05T13:17:17,176 DEBUG [StoreOpener-943cec28fed0bc3f97c304818b208e3f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,178 INFO [StoreOpener-943cec28fed0bc3f97c304818b208e3f-1 {}] regionserver.HStore(327): Store=943cec28fed0bc3f97c304818b208e3f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,178 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 943cec28fed0bc3f97c304818b208e3f 2024-12-05T13:17:17,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=12 2024-12-05T13:17:17,180 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/943cec28fed0bc3f97c304818b208e3f 2024-12-05T13:17:17,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 7bf9cb128da6b1afbc569bd4b25eba2d, server=88aaa70743e7,36055,1733404632807 in 233 msec 2024-12-05T13:17:17,181 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/943cec28fed0bc3f97c304818b208e3f 2024-12-05T13:17:17,182 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 943cec28fed0bc3f97c304818b208e3f 2024-12-05T13:17:17,182 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 943cec28fed0bc3f97c304818b208e3f 2024-12-05T13:17:17,182 INFO [StoreOpener-cabc686ba5a0c762c06d5a4d0024dd7b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:17,183 INFO [StoreOpener-5c9639ed41e308974b4199260665db3c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5c9639ed41e308974b4199260665db3c 2024-12-05T13:17:17,184 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=88aaa70743e7,39293,1733404632706, table=test, region=bb504ea56b81b22b5ca81b650f200ded. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T13:17:17,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7bf9cb128da6b1afbc569bd4b25eba2d, ASSIGN in 568 msec 2024-12-05T13:17:17,186 INFO [StoreOpener-cabc686ba5a0c762c06d5a4d0024dd7b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cabc686ba5a0c762c06d5a4d0024dd7b columnFamilyName family 2024-12-05T13:17:17,186 DEBUG [StoreOpener-cabc686ba5a0c762c06d5a4d0024dd7b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,188 INFO [StoreOpener-cabc686ba5a0c762c06d5a4d0024dd7b-1 {}] regionserver.HStore(327): Store=cabc686ba5a0c762c06d5a4d0024dd7b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,188 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 943cec28fed0bc3f97c304818b208e3f 2024-12-05T13:17:17,188 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:17,188 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:17,191 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:17,193 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:17,194 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:17,194 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/943cec28fed0bc3f97c304818b208e3f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,194 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:17,195 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 943cec28fed0bc3f97c304818b208e3f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61051052, jitterRate=-0.09026843309402466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,196 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 943cec28fed0bc3f97c304818b208e3f 2024-12-05T13:17:17,196 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 943cec28fed0bc3f97c304818b208e3f: Running coprocessor pre-open hook at 1733404637157Writing region info on filesystem at 1733404637157Initializing all the Stores at 1733404637160 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637160Cleaning up temporary data from old regions at 1733404637182 (+22 ms)Running coprocessor post-open hooks at 1733404637196 (+14 ms)Region opened successfully at 1733404637196 2024-12-05T13:17:17,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure abced629a2bc26273048aa2f4b6f8d9c, server=88aaa70743e7,39293,1733404632706 because future has completed 2024-12-05T13:17:17,201 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1733404635844.943cec28fed0bc3f97c304818b208e3f., pid=53, masterSystemTime=1733404637063 2024-12-05T13:17:17,202 INFO [StoreOpener-5c9639ed41e308974b4199260665db3c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5c9639ed41e308974b4199260665db3c columnFamilyName family 2024-12-05T13:17:17,202 DEBUG [StoreOpener-5c9639ed41e308974b4199260665db3c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=23 2024-12-05T13:17:17,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,41495,1733404632848 in 358 msec 2024-12-05T13:17:17,204 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:17,204 INFO [StoreOpener-5c9639ed41e308974b4199260665db3c-1 {}] regionserver.HStore(327): Store=5c9639ed41e308974b4199260665db3c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,205 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 5c9639ed41e308974b4199260665db3c 2024-12-05T13:17:17,205 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1733404635844.943cec28fed0bc3f97c304818b208e3f. 2024-12-05T13:17:17,205 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,ccc,1733404635844.943cec28fed0bc3f97c304818b208e3f. 2024-12-05T13:17:17,206 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,uuu,1733404635844.f4b86b8085620be60c3a91043e8f4d72. 2024-12-05T13:17:17,206 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => f4b86b8085620be60c3a91043e8f4d72, NAME => 'test,uuu,1733404635844.f4b86b8085620be60c3a91043e8f4d72.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-12-05T13:17:17,206 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f4b86b8085620be60c3a91043e8f4d72 2024-12-05T13:17:17,206 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,uuu,1733404635844.f4b86b8085620be60c3a91043e8f4d72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,207 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/5c9639ed41e308974b4199260665db3c 2024-12-05T13:17:17,207 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for f4b86b8085620be60c3a91043e8f4d72 2024-12-05T13:17:17,207 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for f4b86b8085620be60c3a91043e8f4d72 2024-12-05T13:17:17,208 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/5c9639ed41e308974b4199260665db3c 2024-12-05T13:17:17,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN in 592 msec 2024-12-05T13:17:17,208 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 5c9639ed41e308974b4199260665db3c 2024-12-05T13:17:17,209 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 5c9639ed41e308974b4199260665db3c 2024-12-05T13:17:17,209 INFO [StoreOpener-f4b86b8085620be60c3a91043e8f4d72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f4b86b8085620be60c3a91043e8f4d72 2024-12-05T13:17:17,212 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:17,212 INFO [StoreOpener-f4b86b8085620be60c3a91043e8f4d72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4b86b8085620be60c3a91043e8f4d72 columnFamilyName family 2024-12-05T13:17:17,212 DEBUG [StoreOpener-f4b86b8085620be60c3a91043e8f4d72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,213 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 5c9639ed41e308974b4199260665db3c 2024-12-05T13:17:17,213 INFO [StoreOpener-f4b86b8085620be60c3a91043e8f4d72-1 {}] regionserver.HStore(327): Store=f4b86b8085620be60c3a91043e8f4d72/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,213 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for f4b86b8085620be60c3a91043e8f4d72 2024-12-05T13:17:17,215 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/f4b86b8085620be60c3a91043e8f4d72 2024-12-05T13:17:17,216 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/f4b86b8085620be60c3a91043e8f4d72 2024-12-05T13:17:17,216 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 943cec28fed0bc3f97c304818b208e3f, server=88aaa70743e7,36055,1733404632807 because future has completed 2024-12-05T13:17:17,216 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/cabc686ba5a0c762c06d5a4d0024dd7b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,217 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for f4b86b8085620be60c3a91043e8f4d72 2024-12-05T13:17:17,217 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for f4b86b8085620be60c3a91043e8f4d72 2024-12-05T13:17:17,219 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened cabc686ba5a0c762c06d5a4d0024dd7b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61436582, jitterRate=-0.08452358841896057}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,219 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:17,219 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for cabc686ba5a0c762c06d5a4d0024dd7b: Running coprocessor pre-open hook at 1733404637162Writing region info on filesystem at 1733404637162Initializing all the Stores at 1733404637170 (+8 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637170Cleaning up temporary data from old regions at 1733404637194 (+24 ms)Running coprocessor post-open hooks at 1733404637219 (+25 ms)Region opened successfully at 1733404637219 2024-12-05T13:17:17,223 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for f4b86b8085620be60c3a91043e8f4d72 2024-12-05T13:17:17,224 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/5c9639ed41e308974b4199260665db3c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,226 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b., pid=31, masterSystemTime=1733404636942 2024-12-05T13:17:17,229 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 5c9639ed41e308974b4199260665db3c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73532840, jitterRate=0.09572470188140869}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,230 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5c9639ed41e308974b4199260665db3c 2024-12-05T13:17:17,230 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 5c9639ed41e308974b4199260665db3c: Running coprocessor pre-open hook at 1733404637175Writing region info on filesystem at 1733404637175Initializing all the Stores at 1733404637179 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637179Cleaning up temporary data from old regions at 1733404637209 (+30 ms)Running coprocessor post-open hooks at 1733404637230 (+21 ms)Region opened successfully at 1733404637230 2024-12-05T13:17:17,234 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1733404635844.5c9639ed41e308974b4199260665db3c., pid=50, masterSystemTime=1733404636969 2024-12-05T13:17:17,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=27 2024-12-05T13:17:17,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure abced629a2bc26273048aa2f4b6f8d9c, server=88aaa70743e7,39293,1733404632706 in 362 msec 2024-12-05T13:17:17,237 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b. 2024-12-05T13:17:17,237 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b. 2024-12-05T13:17:17,238 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,ppp,1733404635844.3c03197a9f3300f90c023ccedb4f2112. 2024-12-05T13:17:17,238 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 3c03197a9f3300f90c023ccedb4f2112, NAME => 'test,ppp,1733404635844.3c03197a9f3300f90c023ccedb4f2112.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-12-05T13:17:17,238 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3c03197a9f3300f90c023ccedb4f2112 2024-12-05T13:17:17,238 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,ppp,1733404635844.3c03197a9f3300f90c023ccedb4f2112.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,238 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 3c03197a9f3300f90c023ccedb4f2112 2024-12-05T13:17:17,238 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 3c03197a9f3300f90c023ccedb4f2112 2024-12-05T13:17:17,250 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/f4b86b8085620be60c3a91043e8f4d72/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,251 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,41495,1733404632848 2024-12-05T13:17:17,251 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened f4b86b8085620be60c3a91043e8f4d72; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69051813, jitterRate=0.02895219624042511}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,251 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f4b86b8085620be60c3a91043e8f4d72 2024-12-05T13:17:17,252 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for f4b86b8085620be60c3a91043e8f4d72: Running coprocessor pre-open hook at 1733404637207Writing region info on filesystem at 1733404637207Initializing all the Stores at 1733404637208 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637208Cleaning up temporary data from old regions at 1733404637217 (+9 ms)Running coprocessor post-open hooks at 1733404637251 (+34 ms)Region opened successfully at 1733404637251 2024-12-05T13:17:17,255 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1733404635844.f4b86b8085620be60c3a91043e8f4d72., pid=49, masterSystemTime=1733404637063 2024-12-05T13:17:17,257 INFO [StoreOpener-3c03197a9f3300f90c023ccedb4f2112-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3c03197a9f3300f90c023ccedb4f2112 2024-12-05T13:17:17,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure cabc686ba5a0c762c06d5a4d0024dd7b, server=88aaa70743e7,41495,1733404632848 because future has completed 2024-12-05T13:17:17,259 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1733404635844.5c9639ed41e308974b4199260665db3c. 2024-12-05T13:17:17,260 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,,1733404635844.5c9639ed41e308974b4199260665db3c. 2024-12-05T13:17:17,260 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,ggg,1733404635844.4debf2a1857c920c4d806684d7cb0807. 2024-12-05T13:17:17,260 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 4debf2a1857c920c4d806684d7cb0807, NAME => 'test,ggg,1733404635844.4debf2a1857c920c4d806684d7cb0807.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-12-05T13:17:17,261 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4debf2a1857c920c4d806684d7cb0807 2024-12-05T13:17:17,261 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,ggg,1733404635844.4debf2a1857c920c4d806684d7cb0807.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,261 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 4debf2a1857c920c4d806684d7cb0807 2024-12-05T13:17:17,261 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 4debf2a1857c920c4d806684d7cb0807 2024-12-05T13:17:17,265 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1733404635844.f4b86b8085620be60c3a91043e8f4d72. 2024-12-05T13:17:17,266 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,uuu,1733404635844.f4b86b8085620be60c3a91043e8f4d72. 2024-12-05T13:17:17,266 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,xxx,1733404635844.e805a0311eea683702ecc957a61057e2. 2024-12-05T13:17:17,266 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5c9639ed41e308974b4199260665db3c, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:17,266 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => e805a0311eea683702ecc957a61057e2, NAME => 'test,xxx,1733404635844.e805a0311eea683702ecc957a61057e2.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-12-05T13:17:17,266 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e805a0311eea683702ecc957a61057e2 2024-12-05T13:17:17,267 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,xxx,1733404635844.e805a0311eea683702ecc957a61057e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,267 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for e805a0311eea683702ecc957a61057e2 2024-12-05T13:17:17,267 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for e805a0311eea683702ecc957a61057e2 2024-12-05T13:17:17,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5c9639ed41e308974b4199260665db3c, server=88aaa70743e7,39293,1733404632706 because future has completed 2024-12-05T13:17:17,272 INFO [StoreOpener-4debf2a1857c920c4d806684d7cb0807-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4debf2a1857c920c4d806684d7cb0807 2024-12-05T13:17:17,275 INFO [StoreOpener-3c03197a9f3300f90c023ccedb4f2112-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c03197a9f3300f90c023ccedb4f2112 columnFamilyName family 2024-12-05T13:17:17,275 DEBUG [StoreOpener-3c03197a9f3300f90c023ccedb4f2112-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,276 INFO [StoreOpener-e805a0311eea683702ecc957a61057e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e805a0311eea683702ecc957a61057e2 2024-12-05T13:17:17,277 INFO [StoreOpener-3c03197a9f3300f90c023ccedb4f2112-1 {}] regionserver.HStore(327): Store=3c03197a9f3300f90c023ccedb4f2112/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,277 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 3c03197a9f3300f90c023ccedb4f2112 2024-12-05T13:17:17,279 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/3c03197a9f3300f90c023ccedb4f2112 2024-12-05T13:17:17,280 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/3c03197a9f3300f90c023ccedb4f2112 2024-12-05T13:17:17,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-12-05T13:17:17,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure cabc686ba5a0c762c06d5a4d0024dd7b, server=88aaa70743e7,41495,1733404632848 in 485 msec 2024-12-05T13:17:17,281 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 3c03197a9f3300f90c023ccedb4f2112 2024-12-05T13:17:17,281 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 3c03197a9f3300f90c023ccedb4f2112 2024-12-05T13:17:17,282 INFO [StoreOpener-e805a0311eea683702ecc957a61057e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e805a0311eea683702ecc957a61057e2 columnFamilyName family 2024-12-05T13:17:17,282 DEBUG [StoreOpener-e805a0311eea683702ecc957a61057e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,283 INFO [StoreOpener-e805a0311eea683702ecc957a61057e2-1 {}] regionserver.HStore(327): Store=e805a0311eea683702ecc957a61057e2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,284 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for e805a0311eea683702ecc957a61057e2 2024-12-05T13:17:17,284 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 3c03197a9f3300f90c023ccedb4f2112 2024-12-05T13:17:17,285 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/e805a0311eea683702ecc957a61057e2 2024-12-05T13:17:17,287 INFO [StoreOpener-4debf2a1857c920c4d806684d7cb0807-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4debf2a1857c920c4d806684d7cb0807 columnFamilyName family 2024-12-05T13:17:17,288 DEBUG [StoreOpener-4debf2a1857c920c4d806684d7cb0807-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,289 INFO [StoreOpener-4debf2a1857c920c4d806684d7cb0807-1 {}] regionserver.HStore(327): Store=4debf2a1857c920c4d806684d7cb0807/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,289 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 4debf2a1857c920c4d806684d7cb0807 2024-12-05T13:17:17,289 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/e805a0311eea683702ecc957a61057e2 2024-12-05T13:17:17,290 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/4debf2a1857c920c4d806684d7cb0807 2024-12-05T13:17:17,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN in 668 msec 2024-12-05T13:17:17,291 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/4debf2a1857c920c4d806684d7cb0807 2024-12-05T13:17:17,292 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 4debf2a1857c920c4d806684d7cb0807 2024-12-05T13:17:17,292 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for e805a0311eea683702ecc957a61057e2 2024-12-05T13:17:17,292 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 4debf2a1857c920c4d806684d7cb0807 2024-12-05T13:17:17,293 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for e805a0311eea683702ecc957a61057e2 2024-12-05T13:17:17,296 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/3c03197a9f3300f90c023ccedb4f2112/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,298 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 4debf2a1857c920c4d806684d7cb0807 2024-12-05T13:17:17,298 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 3c03197a9f3300f90c023ccedb4f2112; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59268494, jitterRate=-0.11683061718940735}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,298 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3c03197a9f3300f90c023ccedb4f2112 2024-12-05T13:17:17,298 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=88aaa70743e7,39293,1733404632706, table=test, region=bb504ea56b81b22b5ca81b650f200ded. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T13:17:17,298 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 3c03197a9f3300f90c023ccedb4f2112: Running coprocessor pre-open hook at 1733404637239Writing region info on filesystem at 1733404637239Initializing all the Stores at 1733404637241 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637241Cleaning up temporary data from old regions at 1733404637281 (+40 ms)Running coprocessor post-open hooks at 1733404637298 (+17 ms)Region opened successfully at 1733404637298 2024-12-05T13:17:17,300 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for e805a0311eea683702ecc957a61057e2 2024-12-05T13:17:17,303 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1733404635844.3c03197a9f3300f90c023ccedb4f2112., pid=32, masterSystemTime=1733404636942 2024-12-05T13:17:17,307 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=88aaa70743e7,36055,1733404632807, table=test, region=943cec28fed0bc3f97c304818b208e3f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T13:17:17,317 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1733404635844.3c03197a9f3300f90c023ccedb4f2112. 2024-12-05T13:17:17,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=5 2024-12-05T13:17:17,317 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,ppp,1733404635844.3c03197a9f3300f90c023ccedb4f2112. 2024-12-05T13:17:17,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5c9639ed41e308974b4199260665db3c, server=88aaa70743e7,39293,1733404632706 in 371 msec 2024-12-05T13:17:17,318 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5. 2024-12-05T13:17:17,318 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => cb80ccb4b3541210c6fda8b4b988adb5, NAME => 'test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-12-05T13:17:17,318 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:17,319 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,319 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:17,319 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:17,320 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/4debf2a1857c920c4d806684d7cb0807/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,321 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 4debf2a1857c920c4d806684d7cb0807; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64438644, jitterRate=-0.03978937864303589}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,321 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4debf2a1857c920c4d806684d7cb0807 2024-12-05T13:17:17,321 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 4debf2a1857c920c4d806684d7cb0807: Running coprocessor pre-open hook at 1733404637261Writing region info on filesystem at 1733404637261Initializing all the Stores at 1733404637266 (+5 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637266Cleaning up temporary data from old regions at 1733404637293 (+27 ms)Running coprocessor post-open hooks at 1733404637321 (+28 ms)Region opened successfully at 1733404637321 2024-12-05T13:17:17,327 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1733404635844.4debf2a1857c920c4d806684d7cb0807., pid=42, masterSystemTime=1733404636969 2024-12-05T13:17:17,328 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/e805a0311eea683702ecc957a61057e2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,329 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened e805a0311eea683702ecc957a61057e2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60605647, jitterRate=-0.09690548479557037}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,329 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e805a0311eea683702ecc957a61057e2 2024-12-05T13:17:17,329 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for e805a0311eea683702ecc957a61057e2: Running coprocessor pre-open hook at 1733404637267Writing region info on filesystem at 1733404637267Initializing all the Stores at 1733404637269 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637269Cleaning up temporary data from old regions at 1733404637293 (+24 ms)Running coprocessor post-open hooks at 1733404637329 (+36 ms)Region opened successfully at 1733404637329 2024-12-05T13:17:17,331 INFO [StoreOpener-cb80ccb4b3541210c6fda8b4b988adb5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:17,335 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1733404635844.e805a0311eea683702ecc957a61057e2., pid=47, masterSystemTime=1733404637063 2024-12-05T13:17:17,337 INFO [StoreOpener-cb80ccb4b3541210c6fda8b4b988adb5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb80ccb4b3541210c6fda8b4b988adb5 columnFamilyName family 2024-12-05T13:17:17,337 DEBUG [StoreOpener-cb80ccb4b3541210c6fda8b4b988adb5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,338 INFO [StoreOpener-cb80ccb4b3541210c6fda8b4b988adb5-1 {}] regionserver.HStore(327): Store=cb80ccb4b3541210c6fda8b4b988adb5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,339 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:17,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5c9639ed41e308974b4199260665db3c, ASSIGN in 706 msec 2024-12-05T13:17:17,341 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1733404635844.4debf2a1857c920c4d806684d7cb0807. 2024-12-05T13:17:17,341 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,ggg,1733404635844.4debf2a1857c920c4d806684d7cb0807. 2024-12-05T13:17:17,341 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:17,342 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,zzz,1733404635844.6f220ce002468367c756923f6fbc1fc1. 2024-12-05T13:17:17,342 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 6f220ce002468367c756923f6fbc1fc1, NAME => 'test,zzz,1733404635844.6f220ce002468367c756923f6fbc1fc1.', STARTKEY => 'zzz', ENDKEY => ''} 2024-12-05T13:17:17,342 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6f220ce002468367c756923f6fbc1fc1 2024-12-05T13:17:17,343 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,zzz,1733404635844.6f220ce002468367c756923f6fbc1fc1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,343 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 6f220ce002468367c756923f6fbc1fc1 2024-12-05T13:17:17,343 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 6f220ce002468367c756923f6fbc1fc1 2024-12-05T13:17:17,343 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:17,343 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1733404635844.e805a0311eea683702ecc957a61057e2. 2024-12-05T13:17:17,344 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,xxx,1733404635844.e805a0311eea683702ecc957a61057e2. 2024-12-05T13:17:17,344 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,ooo,1733404635844.bd586ec5ee08f004669e3c48942b3638. 2024-12-05T13:17:17,344 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => bd586ec5ee08f004669e3c48942b3638, NAME => 'test,ooo,1733404635844.bd586ec5ee08f004669e3c48942b3638.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-12-05T13:17:17,344 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bd586ec5ee08f004669e3c48942b3638 2024-12-05T13:17:17,344 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,ooo,1733404635844.bd586ec5ee08f004669e3c48942b3638.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,345 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for bd586ec5ee08f004669e3c48942b3638 2024-12-05T13:17:17,345 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for bd586ec5ee08f004669e3c48942b3638 2024-12-05T13:17:17,345 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:17,346 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:17,347 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:17,350 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:17,351 INFO [StoreOpener-6f220ce002468367c756923f6fbc1fc1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6f220ce002468367c756923f6fbc1fc1 2024-12-05T13:17:17,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4debf2a1857c920c4d806684d7cb0807, server=88aaa70743e7,39293,1733404632706 because future has completed 2024-12-05T13:17:17,355 INFO [StoreOpener-bd586ec5ee08f004669e3c48942b3638-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bd586ec5ee08f004669e3c48942b3638 2024-12-05T13:17:17,355 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=e805a0311eea683702ecc957a61057e2, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:17,357 INFO [StoreOpener-6f220ce002468367c756923f6fbc1fc1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f220ce002468367c756923f6fbc1fc1 columnFamilyName family 2024-12-05T13:17:17,357 DEBUG [StoreOpener-6f220ce002468367c756923f6fbc1fc1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,358 INFO [StoreOpener-bd586ec5ee08f004669e3c48942b3638-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd586ec5ee08f004669e3c48942b3638 columnFamilyName family 2024-12-05T13:17:17,359 DEBUG [StoreOpener-bd586ec5ee08f004669e3c48942b3638-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,359 INFO [StoreOpener-6f220ce002468367c756923f6fbc1fc1-1 {}] regionserver.HStore(327): Store=6f220ce002468367c756923f6fbc1fc1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,359 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 6f220ce002468367c756923f6fbc1fc1 2024-12-05T13:17:17,360 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/6f220ce002468367c756923f6fbc1fc1 2024-12-05T13:17:17,361 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/6f220ce002468367c756923f6fbc1fc1 2024-12-05T13:17:17,364 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 6f220ce002468367c756923f6fbc1fc1 2024-12-05T13:17:17,364 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 6f220ce002468367c756923f6fbc1fc1 2024-12-05T13:17:17,364 INFO [StoreOpener-bd586ec5ee08f004669e3c48942b3638-1 {}] regionserver.HStore(327): Store=bd586ec5ee08f004669e3c48942b3638/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,364 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for bd586ec5ee08f004669e3c48942b3638 2024-12-05T13:17:17,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure e805a0311eea683702ecc957a61057e2, server=88aaa70743e7,36055,1733404632807 because future has completed 2024-12-05T13:17:17,365 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/bd586ec5ee08f004669e3c48942b3638 2024-12-05T13:17:17,366 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/bd586ec5ee08f004669e3c48942b3638 2024-12-05T13:17:17,366 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 6f220ce002468367c756923f6fbc1fc1 2024-12-05T13:17:17,367 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for bd586ec5ee08f004669e3c48942b3638 2024-12-05T13:17:17,367 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for bd586ec5ee08f004669e3c48942b3638 2024-12-05T13:17:17,369 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for bd586ec5ee08f004669e3c48942b3638 2024-12-05T13:17:17,369 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/6f220ce002468367c756923f6fbc1fc1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,370 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 6f220ce002468367c756923f6fbc1fc1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73574819, jitterRate=0.09635023772716522}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,370 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6f220ce002468367c756923f6fbc1fc1 2024-12-05T13:17:17,370 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 6f220ce002468367c756923f6fbc1fc1: Running coprocessor pre-open hook at 1733404637343Writing region info on filesystem at 1733404637343Initializing all the Stores at 1733404637345 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637345Cleaning up temporary data from old regions at 1733404637364 (+19 ms)Running coprocessor post-open hooks at 1733404637370 (+6 ms)Region opened successfully at 1733404637370 2024-12-05T13:17:17,370 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/cb80ccb4b3541210c6fda8b4b988adb5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,371 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened cb80ccb4b3541210c6fda8b4b988adb5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74654915, jitterRate=0.1124449223279953}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,372 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:17,372 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for cb80ccb4b3541210c6fda8b4b988adb5: Running coprocessor pre-open hook at 1733404637319Writing region info on filesystem at 1733404637319Initializing all the Stores at 1733404637326 (+7 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637327 (+1 ms)Cleaning up temporary data from old regions at 1733404637347 (+20 ms)Running coprocessor post-open hooks at 1733404637372 (+25 ms)Region opened successfully at 1733404637372 2024-12-05T13:17:17,372 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/bd586ec5ee08f004669e3c48942b3638/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,373 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5., pid=39, masterSystemTime=1733404636942 2024-12-05T13:17:17,373 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened bd586ec5ee08f004669e3c48942b3638; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60857982, jitterRate=-0.09314540028572083}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,374 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bd586ec5ee08f004669e3c48942b3638 2024-12-05T13:17:17,374 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for bd586ec5ee08f004669e3c48942b3638: Running coprocessor pre-open hook at 1733404637345Writing region info on filesystem at 1733404637345Initializing all the Stores at 1733404637347 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637347Cleaning up temporary data from old regions at 1733404637367 (+20 ms)Running coprocessor post-open hooks at 1733404637374 (+7 ms)Region opened successfully at 1733404637374 2024-12-05T13:17:17,375 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1733404635844.bd586ec5ee08f004669e3c48942b3638., pid=51, masterSystemTime=1733404637063 2024-12-05T13:17:17,377 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5. 2024-12-05T13:17:17,377 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5. 2024-12-05T13:17:17,377 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1733404635844.6f220ce002468367c756923f6fbc1fc1., pid=34, masterSystemTime=1733404636969 2024-12-05T13:17:17,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=11 2024-12-05T13:17:17,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 4debf2a1857c920c4d806684d7cb0807, server=88aaa70743e7,39293,1733404632706 in 496 msec 2024-12-05T13:17:17,378 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1733404635844.bd586ec5ee08f004669e3c48942b3638. 2024-12-05T13:17:17,378 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,ooo,1733404635844.bd586ec5ee08f004669e3c48942b3638. 2024-12-05T13:17:17,378 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,kkk,1733404635844.f29f7ad7d2427bd1681491cb1725214f. 2024-12-05T13:17:17,378 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => f29f7ad7d2427bd1681491cb1725214f, NAME => 'test,kkk,1733404635844.f29f7ad7d2427bd1681491cb1725214f.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-12-05T13:17:17,379 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f29f7ad7d2427bd1681491cb1725214f 2024-12-05T13:17:17,379 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,kkk,1733404635844.f29f7ad7d2427bd1681491cb1725214f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,379 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for f29f7ad7d2427bd1681491cb1725214f 2024-12-05T13:17:17,379 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for f29f7ad7d2427bd1681491cb1725214f 2024-12-05T13:17:17,380 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:17,380 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1733404635844.6f220ce002468367c756923f6fbc1fc1. 2024-12-05T13:17:17,380 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,zzz,1733404635844.6f220ce002468367c756923f6fbc1fc1. 2024-12-05T13:17:17,380 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,jjj,1733404635844.8d9d0bbe2316236e0d9afb8d72cd4099. 2024-12-05T13:17:17,381 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 8d9d0bbe2316236e0d9afb8d72cd4099, NAME => 'test,jjj,1733404635844.8d9d0bbe2316236e0d9afb8d72cd4099.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-12-05T13:17:17,381 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8d9d0bbe2316236e0d9afb8d72cd4099 2024-12-05T13:17:17,381 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,jjj,1733404635844.8d9d0bbe2316236e0d9afb8d72cd4099.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,381 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 8d9d0bbe2316236e0d9afb8d72cd4099 2024-12-05T13:17:17,381 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 8d9d0bbe2316236e0d9afb8d72cd4099 2024-12-05T13:17:17,383 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd586ec5ee08f004669e3c48942b3638, server=88aaa70743e7,36055,1733404632807 because future has completed 2024-12-05T13:17:17,384 INFO [StoreOpener-f29f7ad7d2427bd1681491cb1725214f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f29f7ad7d2427bd1681491cb1725214f 2024-12-05T13:17:17,385 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:17,386 INFO [StoreOpener-f29f7ad7d2427bd1681491cb1725214f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f29f7ad7d2427bd1681491cb1725214f columnFamilyName family 2024-12-05T13:17:17,386 DEBUG [StoreOpener-f29f7ad7d2427bd1681491cb1725214f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,387 INFO [StoreOpener-f29f7ad7d2427bd1681491cb1725214f-1 {}] regionserver.HStore(327): Store=f29f7ad7d2427bd1681491cb1725214f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,388 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for f29f7ad7d2427bd1681491cb1725214f 2024-12-05T13:17:17,388 INFO [StoreOpener-8d9d0bbe2316236e0d9afb8d72cd4099-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8d9d0bbe2316236e0d9afb8d72cd4099 2024-12-05T13:17:17,389 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/f29f7ad7d2427bd1681491cb1725214f 2024-12-05T13:17:17,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f220ce002468367c756923f6fbc1fc1, server=88aaa70743e7,39293,1733404632706 because future has completed 2024-12-05T13:17:17,390 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/f29f7ad7d2427bd1681491cb1725214f 2024-12-05T13:17:17,391 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for f29f7ad7d2427bd1681491cb1725214f 2024-12-05T13:17:17,392 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for f29f7ad7d2427bd1681491cb1725214f 2024-12-05T13:17:17,394 INFO [StoreOpener-8d9d0bbe2316236e0d9afb8d72cd4099-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d9d0bbe2316236e0d9afb8d72cd4099 columnFamilyName family 2024-12-05T13:17:17,394 DEBUG [StoreOpener-8d9d0bbe2316236e0d9afb8d72cd4099-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,395 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for f29f7ad7d2427bd1681491cb1725214f 2024-12-05T13:17:17,395 INFO [StoreOpener-8d9d0bbe2316236e0d9afb8d72cd4099-1 {}] regionserver.HStore(327): Store=8d9d0bbe2316236e0d9afb8d72cd4099/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,395 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 8d9d0bbe2316236e0d9afb8d72cd4099 2024-12-05T13:17:17,396 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/8d9d0bbe2316236e0d9afb8d72cd4099 2024-12-05T13:17:17,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=19 2024-12-05T13:17:17,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure bd586ec5ee08f004669e3c48942b3638, server=88aaa70743e7,36055,1733404632807 in 460 msec 2024-12-05T13:17:17,398 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/8d9d0bbe2316236e0d9afb8d72cd4099 2024-12-05T13:17:17,398 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 8d9d0bbe2316236e0d9afb8d72cd4099 2024-12-05T13:17:17,398 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 8d9d0bbe2316236e0d9afb8d72cd4099 2024-12-05T13:17:17,399 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/f29f7ad7d2427bd1681491cb1725214f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,401 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened f29f7ad7d2427bd1681491cb1725214f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63317718, jitterRate=-0.05649247765541077}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,402 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f29f7ad7d2427bd1681491cb1725214f 2024-12-05T13:17:17,402 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for f29f7ad7d2427bd1681491cb1725214f: Running coprocessor pre-open hook at 1733404637380Writing region info on filesystem at 1733404637380Initializing all the Stores at 1733404637382 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637382Cleaning up temporary data from old regions at 1733404637392 (+10 ms)Running coprocessor post-open hooks at 1733404637402 (+10 ms)Region opened successfully at 1733404637402 2024-12-05T13:17:17,403 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 8d9d0bbe2316236e0d9afb8d72cd4099 2024-12-05T13:17:17,404 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1733404635844.f29f7ad7d2427bd1681491cb1725214f., pid=55, masterSystemTime=1733404637063 2024-12-05T13:17:17,404 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=88aaa70743e7,39293,1733404632706, table=test, region=bb504ea56b81b22b5ca81b650f200ded. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T13:17:17,404 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=88aaa70743e7,39293,1733404632706, table=test, region=6f220ce002468367c756923f6fbc1fc1. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T13:17:17,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=bd586ec5ee08f004669e3c48942b3638, ASSIGN in 786 msec 2024-12-05T13:17:17,407 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1733404635844.f29f7ad7d2427bd1681491cb1725214f. 2024-12-05T13:17:17,407 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,kkk,1733404635844.f29f7ad7d2427bd1681491cb1725214f. 2024-12-05T13:17:17,408 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,eee,1733404635844.8e77e301280f5f9879631fbf6a099514. 2024-12-05T13:17:17,408 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 8e77e301280f5f9879631fbf6a099514, NAME => 'test,eee,1733404635844.8e77e301280f5f9879631fbf6a099514.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-12-05T13:17:17,409 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8e77e301280f5f9879631fbf6a099514 2024-12-05T13:17:17,409 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,eee,1733404635844.8e77e301280f5f9879631fbf6a099514.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,409 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 8e77e301280f5f9879631fbf6a099514 2024-12-05T13:17:17,409 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 8e77e301280f5f9879631fbf6a099514 2024-12-05T13:17:17,411 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=88aaa70743e7,36055,1733404632807, table=test, region=943cec28fed0bc3f97c304818b208e3f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T13:17:17,411 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=88aaa70743e7,36055,1733404632807, table=test, region=e805a0311eea683702ecc957a61057e2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T13:17:17,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=30 2024-12-05T13:17:17,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 6f220ce002468367c756923f6fbc1fc1, server=88aaa70743e7,39293,1733404632706 in 594 msec 2024-12-05T13:17:17,415 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/8d9d0bbe2316236e0d9afb8d72cd4099/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6f220ce002468367c756923f6fbc1fc1, ASSIGN in 801 msec 2024-12-05T13:17:17,417 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 8d9d0bbe2316236e0d9afb8d72cd4099; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59111085, jitterRate=-0.11917619407176971}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,417 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8d9d0bbe2316236e0d9afb8d72cd4099 2024-12-05T13:17:17,417 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 8d9d0bbe2316236e0d9afb8d72cd4099: Running coprocessor pre-open hook at 1733404637382Writing region info on filesystem at 1733404637382Initializing all the Stores at 1733404637383 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637383Cleaning up temporary data from old regions at 1733404637398 (+15 ms)Running coprocessor post-open hooks at 1733404637417 (+19 ms)Region opened successfully at 1733404637417 2024-12-05T13:17:17,418 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1733404635844.8d9d0bbe2316236e0d9afb8d72cd4099., pid=41, masterSystemTime=1733404636969 2024-12-05T13:17:17,419 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:17,423 INFO [StoreOpener-8e77e301280f5f9879631fbf6a099514-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8e77e301280f5f9879631fbf6a099514 2024-12-05T13:17:17,425 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1733404635844.8d9d0bbe2316236e0d9afb8d72cd4099. 2024-12-05T13:17:17,425 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,jjj,1733404635844.8d9d0bbe2316236e0d9afb8d72cd4099. 2024-12-05T13:17:17,425 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,fff,1733404635844.1218265d16f004d3f7900dc1adf62958. 2024-12-05T13:17:17,425 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 1218265d16f004d3f7900dc1adf62958, NAME => 'test,fff,1733404635844.1218265d16f004d3f7900dc1adf62958.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-12-05T13:17:17,426 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1218265d16f004d3f7900dc1adf62958 2024-12-05T13:17:17,426 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,fff,1733404635844.1218265d16f004d3f7900dc1adf62958.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,426 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 1218265d16f004d3f7900dc1adf62958 2024-12-05T13:17:17,426 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 1218265d16f004d3f7900dc1adf62958 2024-12-05T13:17:17,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure f29f7ad7d2427bd1681491cb1725214f, server=88aaa70743e7,36055,1733404632807 because future has completed 2024-12-05T13:17:17,427 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:17,428 INFO [StoreOpener-8e77e301280f5f9879631fbf6a099514-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e77e301280f5f9879631fbf6a099514 columnFamilyName family 2024-12-05T13:17:17,428 INFO [StoreOpener-1218265d16f004d3f7900dc1adf62958-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1218265d16f004d3f7900dc1adf62958 2024-12-05T13:17:17,428 DEBUG [StoreOpener-8e77e301280f5f9879631fbf6a099514-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,429 INFO [StoreOpener-8e77e301280f5f9879631fbf6a099514-1 {}] regionserver.HStore(327): Store=8e77e301280f5f9879631fbf6a099514/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,429 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 8e77e301280f5f9879631fbf6a099514 2024-12-05T13:17:17,431 INFO [StoreOpener-1218265d16f004d3f7900dc1adf62958-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1218265d16f004d3f7900dc1adf62958 columnFamilyName family 2024-12-05T13:17:17,431 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/8e77e301280f5f9879631fbf6a099514 2024-12-05T13:17:17,431 DEBUG [StoreOpener-1218265d16f004d3f7900dc1adf62958-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8d9d0bbe2316236e0d9afb8d72cd4099, server=88aaa70743e7,39293,1733404632706 because future has completed 2024-12-05T13:17:17,431 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/8e77e301280f5f9879631fbf6a099514 2024-12-05T13:17:17,432 INFO [StoreOpener-1218265d16f004d3f7900dc1adf62958-1 {}] regionserver.HStore(327): Store=1218265d16f004d3f7900dc1adf62958/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,432 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 1218265d16f004d3f7900dc1adf62958 2024-12-05T13:17:17,433 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 8e77e301280f5f9879631fbf6a099514 2024-12-05T13:17:17,433 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/1218265d16f004d3f7900dc1adf62958 2024-12-05T13:17:17,433 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 8e77e301280f5f9879631fbf6a099514 2024-12-05T13:17:17,434 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/1218265d16f004d3f7900dc1adf62958 2024-12-05T13:17:17,435 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 1218265d16f004d3f7900dc1adf62958 2024-12-05T13:17:17,435 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 1218265d16f004d3f7900dc1adf62958 2024-12-05T13:17:17,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=15 2024-12-05T13:17:17,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure f29f7ad7d2427bd1681491cb1725214f, server=88aaa70743e7,36055,1733404632807 in 477 msec 2024-12-05T13:17:17,437 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 1218265d16f004d3f7900dc1adf62958 2024-12-05T13:17:17,438 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 8e77e301280f5f9879631fbf6a099514 2024-12-05T13:17:17,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f29f7ad7d2427bd1681491cb1725214f, ASSIGN in 824 msec 2024-12-05T13:17:17,441 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/1218265d16f004d3f7900dc1adf62958/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,442 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 1218265d16f004d3f7900dc1adf62958; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71983948, jitterRate=0.07264441251754761}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,442 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1218265d16f004d3f7900dc1adf62958 2024-12-05T13:17:17,442 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 1218265d16f004d3f7900dc1adf62958: Running coprocessor pre-open hook at 1733404637426Writing region info on filesystem at 1733404637426Initializing all the Stores at 1733404637428 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637428Cleaning up temporary data from old regions at 1733404637435 (+7 ms)Running coprocessor post-open hooks at 1733404637442 (+7 ms)Region opened successfully at 1733404637442 2024-12-05T13:17:17,444 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1733404635844.1218265d16f004d3f7900dc1adf62958., pid=46, masterSystemTime=1733404636969 2024-12-05T13:17:17,446 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/8e77e301280f5f9879631fbf6a099514/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,447 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 8e77e301280f5f9879631fbf6a099514; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63569912, jitterRate=-0.05273449420928955}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,447 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8e77e301280f5f9879631fbf6a099514 2024-12-05T13:17:17,447 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1733404635844.1218265d16f004d3f7900dc1adf62958. 2024-12-05T13:17:17,447 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,fff,1733404635844.1218265d16f004d3f7900dc1adf62958. 2024-12-05T13:17:17,447 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 8e77e301280f5f9879631fbf6a099514: Running coprocessor pre-open hook at 1733404637409Writing region info on filesystem at 1733404637409Initializing all the Stores at 1733404637411 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637411Cleaning up temporary data from old regions at 1733404637433 (+22 ms)Running coprocessor post-open hooks at 1733404637447 (+14 ms)Region opened successfully at 1733404637447 2024-12-05T13:17:17,448 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1733404635844.8e77e301280f5f9879631fbf6a099514., pid=54, masterSystemTime=1733404637063 2024-12-05T13:17:17,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=14 2024-12-05T13:17:17,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 8d9d0bbe2316236e0d9afb8d72cd4099, server=88aaa70743e7,39293,1733404632706 in 573 msec 2024-12-05T13:17:17,451 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1733404635844.8e77e301280f5f9879631fbf6a099514. 2024-12-05T13:17:17,451 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,eee,1733404635844.8e77e301280f5f9879631fbf6a099514. 2024-12-05T13:17:17,451 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,lll,1733404635844.3ea448375edac1598b6cdff464d477bf. 2024-12-05T13:17:17,451 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 3ea448375edac1598b6cdff464d477bf, NAME => 'test,lll,1733404635844.3ea448375edac1598b6cdff464d477bf.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-12-05T13:17:17,451 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3ea448375edac1598b6cdff464d477bf 2024-12-05T13:17:17,452 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,lll,1733404635844.3ea448375edac1598b6cdff464d477bf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,452 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 3ea448375edac1598b6cdff464d477bf 2024-12-05T13:17:17,452 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 3ea448375edac1598b6cdff464d477bf 2024-12-05T13:17:17,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8d9d0bbe2316236e0d9afb8d72cd4099, ASSIGN in 837 msec 2024-12-05T13:17:17,454 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:17,454 INFO [StoreOpener-3ea448375edac1598b6cdff464d477bf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3ea448375edac1598b6cdff464d477bf 2024-12-05T13:17:17,456 INFO [StoreOpener-3ea448375edac1598b6cdff464d477bf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ea448375edac1598b6cdff464d477bf columnFamilyName family 2024-12-05T13:17:17,456 DEBUG [StoreOpener-3ea448375edac1598b6cdff464d477bf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,457 INFO [StoreOpener-3ea448375edac1598b6cdff464d477bf-1 {}] regionserver.HStore(327): Store=3ea448375edac1598b6cdff464d477bf/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,457 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 3ea448375edac1598b6cdff464d477bf 2024-12-05T13:17:17,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e77e301280f5f9879631fbf6a099514, server=88aaa70743e7,36055,1733404632807 because future has completed 2024-12-05T13:17:17,458 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:17,458 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/3ea448375edac1598b6cdff464d477bf 2024-12-05T13:17:17,459 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/3ea448375edac1598b6cdff464d477bf 2024-12-05T13:17:17,459 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 3ea448375edac1598b6cdff464d477bf 2024-12-05T13:17:17,459 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 3ea448375edac1598b6cdff464d477bf 2024-12-05T13:17:17,462 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 3ea448375edac1598b6cdff464d477bf 2024-12-05T13:17:17,463 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1218265d16f004d3f7900dc1adf62958, server=88aaa70743e7,39293,1733404632706 because future has completed 2024-12-05T13:17:17,467 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/3ea448375edac1598b6cdff464d477bf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,468 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 3ea448375edac1598b6cdff464d477bf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74755850, jitterRate=0.11394897103309631}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,468 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3ea448375edac1598b6cdff464d477bf 2024-12-05T13:17:17,469 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 3ea448375edac1598b6cdff464d477bf: Running coprocessor pre-open hook at 1733404637452Writing region info on filesystem at 1733404637452Initializing all the Stores at 1733404637453 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637453Cleaning up temporary data from old regions at 1733404637459 (+6 ms)Running coprocessor post-open hooks at 1733404637468 (+9 ms)Region opened successfully at 1733404637469 (+1 ms) 2024-12-05T13:17:17,470 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1733404635844.3ea448375edac1598b6cdff464d477bf., pid=56, masterSystemTime=1733404637063 2024-12-05T13:17:17,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=9 2024-12-05T13:17:17,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 8e77e301280f5f9879631fbf6a099514, server=88aaa70743e7,36055,1733404632807 in 514 msec 2024-12-05T13:17:17,472 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1733404635844.3ea448375edac1598b6cdff464d477bf. 2024-12-05T13:17:17,472 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,lll,1733404635844.3ea448375edac1598b6cdff464d477bf. 2024-12-05T13:17:17,472 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,rrr,1733404635844.a362f9452e2fdf5e9192ce69e889c30d. 2024-12-05T13:17:17,473 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => a362f9452e2fdf5e9192ce69e889c30d, NAME => 'test,rrr,1733404635844.a362f9452e2fdf5e9192ce69e889c30d.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-12-05T13:17:17,473 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a362f9452e2fdf5e9192ce69e889c30d 2024-12-05T13:17:17,473 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,rrr,1733404635844.a362f9452e2fdf5e9192ce69e889c30d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:17,473 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for a362f9452e2fdf5e9192ce69e889c30d 2024-12-05T13:17:17,473 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for a362f9452e2fdf5e9192ce69e889c30d 2024-12-05T13:17:17,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8e77e301280f5f9879631fbf6a099514, ASSIGN in 859 msec 2024-12-05T13:17:17,476 INFO [StoreOpener-a362f9452e2fdf5e9192ce69e889c30d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a362f9452e2fdf5e9192ce69e889c30d 2024-12-05T13:17:17,476 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:17,478 INFO [StoreOpener-a362f9452e2fdf5e9192ce69e889c30d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a362f9452e2fdf5e9192ce69e889c30d columnFamilyName family 2024-12-05T13:17:17,479 DEBUG [StoreOpener-a362f9452e2fdf5e9192ce69e889c30d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:17,479 INFO [StoreOpener-a362f9452e2fdf5e9192ce69e889c30d-1 {}] regionserver.HStore(327): Store=a362f9452e2fdf5e9192ce69e889c30d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:17,480 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for a362f9452e2fdf5e9192ce69e889c30d 2024-12-05T13:17:17,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3ea448375edac1598b6cdff464d477bf, server=88aaa70743e7,36055,1733404632807 because future has completed 2024-12-05T13:17:17,482 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/a362f9452e2fdf5e9192ce69e889c30d 2024-12-05T13:17:17,482 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/a362f9452e2fdf5e9192ce69e889c30d 2024-12-05T13:17:17,483 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for a362f9452e2fdf5e9192ce69e889c30d 2024-12-05T13:17:17,484 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for a362f9452e2fdf5e9192ce69e889c30d 2024-12-05T13:17:17,487 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for a362f9452e2fdf5e9192ce69e889c30d 2024-12-05T13:17:17,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=10 2024-12-05T13:17:17,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 1218265d16f004d3f7900dc1adf62958, server=88aaa70743e7,39293,1733404632706 in 579 msec 2024-12-05T13:17:17,492 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/a362f9452e2fdf5e9192ce69e889c30d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T13:17:17,493 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened a362f9452e2fdf5e9192ce69e889c30d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66496919, jitterRate=-0.009118691086769104}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:17,493 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a362f9452e2fdf5e9192ce69e889c30d 2024-12-05T13:17:17,494 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for a362f9452e2fdf5e9192ce69e889c30d: Running coprocessor pre-open hook at 1733404637473Writing region info on filesystem at 1733404637473Initializing all the Stores at 1733404637475 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404637475Cleaning up temporary data from old regions at 1733404637484 (+9 ms)Running coprocessor post-open hooks at 1733404637493 (+9 ms)Region opened successfully at 1733404637493 2024-12-05T13:17:17,495 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1733404635844.a362f9452e2fdf5e9192ce69e889c30d., pid=48, masterSystemTime=1733404637063 2024-12-05T13:17:17,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1218265d16f004d3f7900dc1adf62958, ASSIGN in 877 msec 2024-12-05T13:17:17,499 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1733404635844.a362f9452e2fdf5e9192ce69e889c30d. 2024-12-05T13:17:17,499 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,rrr,1733404635844.a362f9452e2fdf5e9192ce69e889c30d. 2024-12-05T13:17:17,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=16 2024-12-05T13:17:17,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 3ea448375edac1598b6cdff464d477bf, server=88aaa70743e7,36055,1733404632807 in 536 msec 2024-12-05T13:17:17,509 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=88aaa70743e7,39293,1733404632706, table=test, region=bb504ea56b81b22b5ca81b650f200ded. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T13:17:17,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3ea448375edac1598b6cdff464d477bf, ASSIGN in 891 msec 2024-12-05T13:17:17,511 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:17,515 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=88aaa70743e7,36055,1733404632807, table=test, region=943cec28fed0bc3f97c304818b208e3f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T13:17:17,516 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=88aaa70743e7,36055,1733404632807, table=test, region=a362f9452e2fdf5e9192ce69e889c30d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T13:17:17,516 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=88aaa70743e7,36055,1733404632807, table=test, region=e805a0311eea683702ecc957a61057e2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T13:17:17,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure a362f9452e2fdf5e9192ce69e889c30d, server=88aaa70743e7,36055,1733404632807 because future has completed 2024-12-05T13:17:17,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4debf2a1857c920c4d806684d7cb0807, ASSIGN in 903 msec 2024-12-05T13:17:17,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=22 2024-12-05T13:17:17,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure a362f9452e2fdf5e9192ce69e889c30d, server=88aaa70743e7,36055,1733404632807 in 605 msec 2024-12-05T13:17:17,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a362f9452e2fdf5e9192ce69e889c30d, ASSIGN in 914 msec 2024-12-05T13:17:17,531 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,41495,1733404632848 2024-12-05T13:17:17,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb80ccb4b3541210c6fda8b4b988adb5, server=88aaa70743e7,41495,1733404632848 because future has completed 2024-12-05T13:17:17,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=28 2024-12-05T13:17:17,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure e805a0311eea683702ecc957a61057e2, server=88aaa70743e7,36055,1733404632807 in 625 msec 2024-12-05T13:17:17,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e805a0311eea683702ecc957a61057e2, ASSIGN in 926 msec 2024-12-05T13:17:17,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=26 2024-12-05T13:17:17,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure cb80ccb4b3541210c6fda8b4b988adb5, server=88aaa70743e7,41495,1733404632848 in 697 msec 2024-12-05T13:17:17,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN in 934 msec 2024-12-05T13:17:17,551 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=3c03197a9f3300f90c023ccedb4f2112, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,41495,1733404632848 2024-12-05T13:17:17,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c03197a9f3300f90c023ccedb4f2112, server=88aaa70743e7,41495,1733404632848 because future has completed 2024-12-05T13:17:17,556 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, openSeqNum=2, regionLocation=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:17,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure f4b86b8085620be60c3a91043e8f4d72, server=88aaa70743e7,36055,1733404632807 because future has completed 2024-12-05T13:17:17,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=20 2024-12-05T13:17:17,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 3c03197a9f3300f90c023ccedb4f2112, server=88aaa70743e7,41495,1733404632848 in 764 msec 2024-12-05T13:17:17,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN in 952 msec 2024-12-05T13:17:17,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=25 2024-12-05T13:17:17,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure f4b86b8085620be60c3a91043e8f4d72, server=88aaa70743e7,36055,1733404632807 in 648 msec 2024-12-05T13:17:17,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f4b86b8085620be60c3a91043e8f4d72, ASSIGN in 960 msec 2024-12-05T13:17:17,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=abced629a2bc26273048aa2f4b6f8d9c, ASSIGN in 963 msec 2024-12-05T13:17:17,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=7 2024-12-05T13:17:17,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 943cec28fed0bc3f97c304818b208e3f, server=88aaa70743e7,36055,1733404632807 in 634 msec 2024-12-05T13:17:17,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=943cec28fed0bc3f97c304818b208e3f, ASSIGN in 970 msec 2024-12-05T13:17:17,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN in 973 msec 2024-12-05T13:17:17,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=24 2024-12-05T13:17:17,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure bb504ea56b81b22b5ca81b650f200ded, server=88aaa70743e7,39293,1733404632706 in 698 msec 2024-12-05T13:17:17,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=4 2024-12-05T13:17:17,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=bb504ea56b81b22b5ca81b650f200ded, ASSIGN in 982 msec 2024-12-05T13:17:17,600 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T13:17:17,600 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733404637600"}]},"ts":"1733404637600"} 2024-12-05T13:17:17,603 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-12-05T13:17:17,605 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T13:17:17,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.7340 sec 2024-12-05T13:17:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T13:17:18,059 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-12-05T13:17:18,059 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-12-05T13:17:18,061 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T13:17:18,079 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-12-05T13:17:18,080 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T13:17:18,081 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-12-05T13:17:18,081 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T13:17:18,089 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-12-05T13:17:18,112 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-12-05T13:17:18,125 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T13:17:18,128 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T13:17:18,132 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T13:17:18,135 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T13:17:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 943cec28fed0bc3f97c304818b208e3f changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8e77e301280f5f9879631fbf6a099514 changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7bf9cb128da6b1afbc569bd4b25eba2d changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f29f7ad7d2427bd1681491cb1725214f changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3ea448375edac1598b6cdff464d477bf changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bd586ec5ee08f004669e3c48942b3638 changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a362f9452e2fdf5e9192ce69e889c30d changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f4b86b8085620be60c3a91043e8f4d72 changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e805a0311eea683702ecc957a61057e2 changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5c9639ed41e308974b4199260665db3c changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1218265d16f004d3f7900dc1adf62958 changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4debf2a1857c920c4d806684d7cb0807 changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8d9d0bbe2316236e0d9afb8d72cd4099 changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c086578586a3011e8fde7b2976cca84b changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5d414b07d03c5766c12220a94662f4ff changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bb504ea56b81b22b5ca81b650f200ded changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region abced629a2bc26273048aa2f4b6f8d9c changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6f220ce002468367c756923f6fbc1fc1 changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8b3cc71809f62cd16a817f5a172f68ff changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f8705d72ae6f07bbf8fa8bfbf5a81d07 changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 01ef88de67b81e433e4128dc0a819605 changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region eb2ca1f66d6c9b4513c79bf39e4e5bfe changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3c03197a9f3300f90c023ccedb4f2112 changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 97426aa8ec496bfcaaf589a5cbe6f1b3 changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cb80ccb4b3541210c6fda8b4b988adb5 changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cabc686ba5a0c762c06d5a4d0024dd7b changed from -1.0 to 0.0, refreshing cache 2024-12-05T13:17:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-12-05T13:17:18,200 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-12-05T13:17:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-12-05T13:17:18,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.BalancerClusterState(204): Hosts are {88aaa70743e7=0} racks are {/default-rack=0} 2024-12-05T13:17:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-12-05T13:17:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-12-05T13:17:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-12-05T13:17:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T13:17:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T13:17:18,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-05T13:17:18,201 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T13:17:18,202 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T13:17:18,202 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-05T13:17:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-12-05T13:17:18,212 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-12-05T13:17:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-12-05T13:17:18,214 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T13:17:18,224 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.3 set balanceSwitch=false 2024-12-05T13:17:18,243 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=381, OpenFileDescriptor=643, MaxFileDescriptor=1048576, SystemLoadAverage=382, ProcessCount=11, AvailableMemoryMB=4546 2024-12-05T13:17:18,246 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-12-05T13:17:18,247 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-12-05T13:17:18,247 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 88aaa70743e7,41495,1733404632848 2024-12-05T13:17:18,247 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 88aaa70743e7,41495,1733404632848: Simulated kill ***** 2024-12-05T13:17:18,247 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-05T13:17:18,249 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-05T13:17:18,251 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-05T13:17:18,252 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-05T13:17:18,252 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-05T13:17:18,252 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 302914160 }, "NonHeapMemoryUsage": { "committed": 152961024, "init": 7667712, "max": -1, "used": 150462848 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "88aaa70743e7" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "88aaa70743e7" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "88aaa70743e7" } ] } 2024-12-05T13:17:18,255 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42815 {}] master.MasterRpcServices(700): 88aaa70743e7,41495,1733404632848 reported a fatal error: ***** ABORTING region server 88aaa70743e7,41495,1733404632848: Simulated kill ***** 2024-12-05T13:17:18,257 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '88aaa70743e7,41495,1733404632848' ***** 2024-12-05T13:17:18,257 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-12-05T13:17:18,257 INFO [RS:2;88aaa70743e7:41495 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T13:17:18,257 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T13:17:18,257 INFO [RS:2;88aaa70743e7:41495 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-05T13:17:18,257 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T13:17:18,258 INFO [RS:2;88aaa70743e7:41495 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-05T13:17:18,258 INFO [RS:2;88aaa70743e7:41495 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T13:17:18,258 INFO [RS:2;88aaa70743e7:41495 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;88aaa70743e7:41495. 2024-12-05T13:17:18,258 DEBUG [RS:2;88aaa70743e7:41495 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T13:17:18,264 DEBUG [RS:2;88aaa70743e7:41495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T13:17:18,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741836_1012 (size=32114) 2024-12-05T13:17:18,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741836_1012 (size=32114) 2024-12-05T13:17:18,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741836_1012 (size=32114) 2024-12-05T13:17:18,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741834_1010 (size=2237) 2024-12-05T13:17:18,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741834_1010 (size=2237) 2024-12-05T13:17:18,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741834_1010 (size=2237) 2024-12-05T13:17:18,276 DEBUG [RS:2;88aaa70743e7:41495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T13:17:18,276 INFO [RS:2;88aaa70743e7:41495 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T13:17:18,276 INFO [RS:2;88aaa70743e7:41495 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41495 2024-12-05T13:17:18,295 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41495-0x101a6d1666f0003, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/88aaa70743e7,41495,1733404632848 2024-12-05T13:17:18,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T13:17:18,295 INFO [RS:2;88aaa70743e7:41495 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T13:17:18,296 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [88aaa70743e7,41495,1733404632848] 2024-12-05T13:17:18,311 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/88aaa70743e7,41495,1733404632848 already deleted, retry=false 2024-12-05T13:17:18,312 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 88aaa70743e7,41495,1733404632848 on 88aaa70743e7,42815,1733404631821 2024-12-05T13:17:18,316 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:18,320 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for 88aaa70743e7,41495,1733404632848 (carryingMeta=true) 88aaa70743e7,41495,1733404632848/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2a088186[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-05T13:17:18,321 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:18,324 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true, isMeta: true 2024-12-05T13:17:18,326 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting 2024-12-05T13:17:18,328 INFO [PEWorker-1 {}] master.SplitWALManager(105): 88aaa70743e7,41495,1733404632848 WAL count=1, meta=true 2024-12-05T13:17:18,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta}] 2024-12-05T13:17:18,334 INFO [regionserver/88aaa70743e7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T13:17:18,336 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:18,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706}] 2024-12-05T13:17:18,359 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T13:17:18,404 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41495-0x101a6d1666f0003, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T13:17:18,404 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41495-0x101a6d1666f0003, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T13:17:18,405 INFO [RS:2;88aaa70743e7:41495 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T13:17:18,405 INFO [RS:2;88aaa70743e7:41495 {}] regionserver.HRegionServer(1031): Exiting; stopping=88aaa70743e7,41495,1733404632848; zookeeper connection closed. 2024-12-05T13:17:18,406 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@29dedd9b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@29dedd9b 2024-12-05T13:17:18,433 INFO [regionserver/88aaa70743e7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-05T13:17:18,433 INFO [regionserver/88aaa70743e7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-05T13:17:18,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-12-05T13:17:18,521 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, size=31.4 K (32114bytes) 2024-12-05T13:17:18,522 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-05T13:17:18,522 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-05T13:17:18,522 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta 2024-12-05T13:17:18,527 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta after 4ms 2024-12-05T13:17:18,535 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta took 15ms 2024-12-05T13:17:18,546 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-05T13:17:18,553 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta.temp 2024-12-05T13:17:18,553 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta so closing down 2024-12-05T13:17:18,554 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-05T13:17:18,559 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta.temp 2024-12-05T13:17:18,562 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-12-05T13:17:18,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741864_1040 (size=31873) 2024-12-05T13:17:18,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741864_1040 (size=31873) 2024-12-05T13:17:18,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741864_1040 (size=31873) 2024-12-05T13:17:18,979 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta.temp (wrote 57 edits, skipped 0 edits in 1 ms) 2024-12-05T13:17:18,982 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta.temp to hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-05T13:17:18,988 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 445 ms; skipped=1; WAL=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-12-05T13:17:18,988 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, journal: Splitting hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, size=31.4 K (32114bytes) at 1733404638521Finishing writing output for hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta so closing down at 1733404638554 (+33 ms)Creating recovered edits writer path=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta.temp at 1733404638559 (+5 ms)3 split writer threads finished at 1733404638562 (+3 ms)Closed recovered edits writer path=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta.temp (wrote 57 edits, skipped 0 edits in 1 ms) at 1733404638979 (+417 ms)Rename recovered edits hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta.temp to hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1733404638982 (+3 ms)Processed 58 edits across 1 Regions in 445 ms; skipped=1; WAL=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1733404638988 (+6 ms) 2024-12-05T13:17:18,988 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta 2024-12-05T13:17:18,990 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-05T13:17:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-05T13:17:18,999 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta to hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/oldWALs 2024-12-05T13:17:19,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-05T13:17:19,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 in 660 msec 2024-12-05T13:17:19,004 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:19,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-05T13:17:19,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 in 675 msec 2024-12-05T13:17:19,009 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting dir is empty, no logs to split. 2024-12-05T13:17:19,010 INFO [PEWorker-1 {}] master.SplitWALManager(105): 88aaa70743e7,41495,1733404632848 WAL count=0, meta=true 2024-12-05T13:17:19,010 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 88aaa70743e7,41495,1733404632848 WAL splitting is done? wals=0, meta=true 2024-12-05T13:17:19,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T13:17:19,013 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:19,013 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T13:17:19,164 DEBUG [88aaa70743e7:42815 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-05T13:17:19,164 DEBUG [88aaa70743e7:42815 {}] balancer.BalancerClusterState(204): Hosts are {88aaa70743e7=0} racks are {/default-rack=0} 2024-12-05T13:17:19,165 DEBUG [88aaa70743e7:42815 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T13:17:19,165 DEBUG [88aaa70743e7:42815 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T13:17:19,165 DEBUG [88aaa70743e7:42815 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T13:17:19,165 DEBUG [88aaa70743e7:42815 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T13:17:19,165 INFO [88aaa70743e7:42815 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T13:17:19,165 INFO [88aaa70743e7:42815 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T13:17:19,165 DEBUG [88aaa70743e7:42815 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T13:17:19,166 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:19,168 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPENING 2024-12-05T13:17:19,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:19,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:19,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:19,208 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:19,208 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:19,208 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:19,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706}] 2024-12-05T13:17:19,208 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:19,366 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T13:17:19,366 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T13:17:19,366 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T13:17:19,369 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=88aaa70743e7%2C39293%2C1733404632706.meta, suffix=.meta, logDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,39293,1733404632706, archiveDir=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/oldWALs, maxLogs=32 2024-12-05T13:17:19,381 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,39293,1733404632706/88aaa70743e7%2C39293%2C1733404632706.meta.1733404639370.meta, exclude list is [], retry=0 2024-12-05T13:17:19,384 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34109,DS-d2ff1f2a-33de-4af6-9d9e-1d533285e26a,DISK] 2024-12-05T13:17:19,384 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42229,DS-0c343192-22cb-4379-b3b1-2cf2bdfac1b3,DISK] 2024-12-05T13:17:19,384 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32831,DS-da32b628-72dd-49f3-b291-1fa8797500ab,DISK] 2024-12-05T13:17:19,387 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,39293,1733404632706/88aaa70743e7%2C39293%2C1733404632706.meta.1733404639370.meta 2024-12-05T13:17:19,388 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33987:33987),(127.0.0.1/127.0.0.1:40917:40917),(127.0.0.1/127.0.0.1:34519:34519)] 2024-12-05T13:17:19,388 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T13:17:19,388 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T13:17:19,388 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T13:17:19,389 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T13:17:19,389 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T13:17:19,389 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:19,389 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T13:17:19,389 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T13:17:19,391 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T13:17:19,392 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T13:17:19,392 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:19,392 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:17:19,393 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T13:17:19,394 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T13:17:19,394 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:19,394 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:17:19,394 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T13:17:19,395 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T13:17:19,395 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:19,396 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:17:19,396 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T13:17:19,397 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T13:17:19,397 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:19,397 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T13:17:19,397 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T13:17:19,398 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740 2024-12-05T13:17:19,401 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740 2024-12-05T13:17:19,402 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-05T13:17:19,416 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-05T13:17:19,419 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-12-05T13:17:19,500 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/.tmp/info/1c077ac7c9d14e0588e27020c56fb926 is 138, key is test,bbb,1733404635844.8b3cc71809f62cd16a817f5a172f68ff./info:regioninfo/1733404637130/Put/seqid=0 2024-12-05T13:17:19,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741866_1042 (size=34924) 2024-12-05T13:17:19,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741866_1042 (size=34924) 2024-12-05T13:17:19,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741866_1042 (size=34924) 2024-12-05T13:17:19,535 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/.tmp/info/1c077ac7c9d14e0588e27020c56fb926 2024-12-05T13:17:19,600 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/.tmp/ns/003c668c5ad944d29dbe5a9c20556429 is 43, key is default/ns:d/1733404635468/Put/seqid=0 2024-12-05T13:17:19,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741867_1043 (size=5153) 2024-12-05T13:17:19,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741867_1043 (size=5153) 2024-12-05T13:17:19,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741867_1043 (size=5153) 2024-12-05T13:17:19,610 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/.tmp/ns/003c668c5ad944d29dbe5a9c20556429 2024-12-05T13:17:19,634 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/.tmp/table/d8de7c5d5bd44799948730209b09df83 is 40, key is test/table:state/1733404637600/Put/seqid=0 2024-12-05T13:17:19,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741868_1044 (size=5165) 2024-12-05T13:17:19,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741868_1044 (size=5165) 2024-12-05T13:17:19,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741868_1044 (size=5165) 2024-12-05T13:17:19,644 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/.tmp/table/d8de7c5d5bd44799948730209b09df83 2024-12-05T13:17:19,653 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/.tmp/info/1c077ac7c9d14e0588e27020c56fb926 as hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/info/1c077ac7c9d14e0588e27020c56fb926 2024-12-05T13:17:19,663 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/info/1c077ac7c9d14e0588e27020c56fb926, entries=260, sequenceid=60, filesize=34.1 K 2024-12-05T13:17:19,666 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/.tmp/ns/003c668c5ad944d29dbe5a9c20556429 as hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/ns/003c668c5ad944d29dbe5a9c20556429 2024-12-05T13:17:19,675 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/ns/003c668c5ad944d29dbe5a9c20556429, entries=2, sequenceid=60, filesize=5.0 K 2024-12-05T13:17:19,677 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/.tmp/table/d8de7c5d5bd44799948730209b09df83 as hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/table/d8de7c5d5bd44799948730209b09df83 2024-12-05T13:17:19,686 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/table/d8de7c5d5bd44799948730209b09df83, entries=2, sequenceid=60, filesize=5.0 K 2024-12-05T13:17:19,689 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 269ms, sequenceid=60, compaction requested=false; wal=null 2024-12-05T13:17:19,691 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-05T13:17:19,694 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-12-05T13:17:19,696 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T13:17:19,696 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T13:17:19,697 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T13:17:19,699 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T13:17:19,702 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-12-05T13:17:19,703 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68489925, jitterRate=0.020579412579536438}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T13:17:19,703 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T13:17:19,703 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733404639389Writing region info on filesystem at 1733404639389Initializing all the Stores at 1733404639390 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733404639390Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733404639391 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404639391Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733404639391Obtaining lock to block concurrent updates at 1733404639419 (+28 ms)Preparing flush snapshotting stores in 1588230740 at 1733404639419Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1733404639422 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733404639422Flushing 1588230740/info: creating writer at 1733404639423 (+1 ms)Flushing 1588230740/info: appending metadata at 1733404639484 (+61 ms)Flushing 1588230740/info: closing flushed file at 1733404639488 (+4 ms)Flushing 1588230740/ns: creating writer at 1733404639585 (+97 ms)Flushing 1588230740/ns: appending metadata at 1733404639599 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733404639599Flushing 1588230740/table: creating writer at 1733404639618 (+19 ms)Flushing 1588230740/table: appending metadata at 1733404639633 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733404639634 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7340e845: reopening flushed file at 1733404639652 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fdb27ad: reopening flushed file at 1733404639665 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4381f83: reopening flushed file at 1733404639676 (+11 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 269ms, sequenceid=60, compaction requested=false; wal=null at 1733404639690 (+14 ms)Cleaning up temporary data from old regions at 1733404639696 (+6 ms)Running coprocessor post-open hooks at 1733404639703 (+7 ms)Region opened successfully at 1733404639703 2024-12-05T13:17:19,704 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1733404639361 2024-12-05T13:17:19,707 DEBUG [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T13:17:19,707 INFO [RS_OPEN_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T13:17:19,708 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:19,710 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:19,724 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-05T13:17:19,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:19,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:19,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:19,762 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:19,762 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:19,762 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:19,763 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:19,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-05T13:17:19,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 in 554 msec 2024-12-05T13:17:19,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-12-05T13:17:19,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 755 msec 2024-12-05T13:17:19,776 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 88aaa70743e7,41495,1733404632848 had 9 regions 2024-12-05T13:17:19,779 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true, isMeta: false 2024-12-05T13:17:19,781 INFO [PEWorker-1 {}] master.SplitWALManager(105): 88aaa70743e7,41495,1733404632848 WAL count=1, meta=false 2024-12-05T13:17:19,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576}] 2024-12-05T13:17:19,784 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:19,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706}] 2024-12-05T13:17:19,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39293 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-12-05T13:17:19,955 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.1733404634576, size=2.2 K (2237bytes) 2024-12-05T13:17:19,956 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.1733404634576 2024-12-05T13:17:19,956 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.1733404634576 after 0ms 2024-12-05T13:17:19,960 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.1733404634576 took 5ms 2024-12-05T13:17:19,965 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for eb2ca1f66d6c9b4513c79bf39e4e5bfe: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-05T13:17:19,967 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for f8705d72ae6f07bbf8fa8bfbf5a81d07: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-05T13:17:19,969 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 01ef88de67b81e433e4128dc0a819605: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-05T13:17:19,971 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 8b3cc71809f62cd16a817f5a172f68ff: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-05T13:17:19,974 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 97426aa8ec496bfcaaf589a5cbe6f1b3: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-05T13:17:19,976 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for cabc686ba5a0c762c06d5a4d0024dd7b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-05T13:17:19,979 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 3c03197a9f3300f90c023ccedb4f2112: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-05T13:17:19,981 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for cb80ccb4b3541210c6fda8b4b988adb5: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-12-05T13:17:19,981 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.1733404634576 so closing down 2024-12-05T13:17:19,981 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-05T13:17:19,981 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-12-05T13:17:19,981 INFO [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 20 ms; skipped=8; WAL=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.1733404634576, size=2.2 K, length=2237, corrupted=false, cancelled=false 2024-12-05T13:17:19,981 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.1733404634576, journal: Splitting hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.1733404634576, size=2.2 K (2237bytes) at 1733404639955Finishing writing output for hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.1733404634576 so closing down at 1733404639981 (+26 ms)3 split writer threads finished at 1733404639981Processed 8 edits across 0 Regions in 20 ms; skipped=8; WAL=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.1733404634576, size=2.2 K, length=2237, corrupted=false, cancelled=false at 1733404639981 2024-12-05T13:17:19,981 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.1733404634576 2024-12-05T13:17:19,981 DEBUG [RS_LOG_REPLAY_OPS-regionserver/88aaa70743e7:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-05T13:17:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42815 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-05T13:17:19,988 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting/88aaa70743e7%2C41495%2C1733404632848.1733404634576 to hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/oldWALs 2024-12-05T13:17:19,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-05T13:17:19,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 in 203 msec 2024-12-05T13:17:19,993 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:19,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-12-05T13:17:19,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 in 212 msec 2024-12-05T13:17:19,998 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/WALs/88aaa70743e7,41495,1733404632848-splitting dir is empty, no logs to split. 2024-12-05T13:17:19,998 INFO [PEWorker-1 {}] master.SplitWALManager(105): 88aaa70743e7,41495,1733404632848 WAL count=0, meta=false 2024-12-05T13:17:19,998 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 88aaa70743e7,41495,1733404632848 WAL splitting is done? wals=0, meta=false 2024-12-05T13:17:20,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN}] 2024-12-05T13:17:20,004 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN 2024-12-05T13:17:20,006 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T13:17:20,007 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:20,008 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T13:17:20,009 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:20,011 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T13:17:20,012 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:20,013 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T13:17:20,014 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:20,015 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T13:17:20,016 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:20,018 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T13:17:20,019 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:20,020 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T13:17:20,021 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:20,022 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T13:17:20,156 DEBUG [88aaa70743e7:42815 {}] balancer.BalancerClusterState(204): Hosts are {88aaa70743e7=0} racks are {/default-rack=0} 2024-12-05T13:17:20,157 DEBUG [88aaa70743e7:42815 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T13:17:20,157 DEBUG [88aaa70743e7:42815 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T13:17:20,157 DEBUG [88aaa70743e7:42815 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T13:17:20,157 DEBUG [88aaa70743e7:42815 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T13:17:20,157 INFO [88aaa70743e7:42815 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T13:17:20,157 INFO [88aaa70743e7:42815 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T13:17:20,157 DEBUG [88aaa70743e7:42815 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T13:17:20,157 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, regionLocation=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:20,158 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, regionLocation=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:20,159 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, regionLocation=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:20,159 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 88aaa70743e7:41495 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 88aaa70743e7/172.17.0.3:41495 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:17:20,159 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, regionLocation=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:20,160 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,161 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,161 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,41495,1733404632848, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,41495,1733404632848, seqNum=-1, error=java.net.ConnectException: Call to address=88aaa70743e7:41495 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 88aaa70743e7/172.17.0.3:41495 2024-12-05T13:17:20,162 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,41495,1733404632848, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-05T13:17:20,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,41495,1733404632848, seqNum=-1 from cache 2024-12-05T13:17:20,162 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, regionLocation=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,163 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T13:17:20,163 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,39293,1733404632706, seqNum=-1] 2024-12-05T13:17:20,163 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T13:17:20,164 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 88aaa70743e7:41495 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 88aaa70743e7/172.17.0.3:41495 2024-12-05T13:17:20,165 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 88aaa70743e7:41495 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 88aaa70743e7/172.17.0.3:41495 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:17:20,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,41495,1733404632848, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,39293,1733404632706, seqNum=-1, error=java.net.ConnectException: Call to address=88aaa70743e7:41495 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 88aaa70743e7/172.17.0.3:41495 2024-12-05T13:17:20,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54953, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T13:17:20,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,41495,1733404632848, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,39293,1733404632706, seqNum=-1, error=java.net.ConnectException: Call to address=88aaa70743e7:41495 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 88aaa70743e7/172.17.0.3:41495 2024-12-05T13:17:20,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,41495,1733404632848, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,39293,1733404632706, seqNum=-1, error=java.net.ConnectException: Call to address=88aaa70743e7:41495 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 88aaa70743e7/172.17.0.3:41495 2024-12-05T13:17:20,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,41495,1733404632848, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,39293,1733404632706, seqNum=-1, error=java.net.ConnectException: Call to address=88aaa70743e7:41495 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 88aaa70743e7/172.17.0.3:41495 2024-12-05T13:17:20,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,41495,1733404632848, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,39293,1733404632706, seqNum=-1, error=java.net.ConnectException: Call to address=88aaa70743e7:41495 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 88aaa70743e7/172.17.0.3:41495 2024-12-05T13:17:20,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,41495,1733404632848, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=88aaa70743e7,39293,1733404632706, seqNum=-1, error=java.net.ConnectException: Call to address=88aaa70743e7:41495 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 88aaa70743e7/172.17.0.3:41495 2024-12-05T13:17:20,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 88aaa70743e7:41495 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 88aaa70743e7/172.17.0.3:41495 2024-12-05T13:17:20,168 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN because future has completed 2024-12-05T13:17:20,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8705d72ae6f07bbf8fa8bfbf5a81d07, server=88aaa70743e7,39293,1733404632706}] 2024-12-05T13:17:20,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN because future has completed 2024-12-05T13:17:20,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb80ccb4b3541210c6fda8b4b988adb5, server=88aaa70743e7,36055,1733404632807}] 2024-12-05T13:17:20,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN because future has completed 2024-12-05T13:17:20,272 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN because future has completed 2024-12-05T13:17:20,281 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN because future has completed 2024-12-05T13:17:20,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN because future has completed 2024-12-05T13:17:20,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN because future has completed 2024-12-05T13:17:20,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706}] 2024-12-05T13:17:20,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN because future has completed 2024-12-05T13:17:20,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb2ca1f66d6c9b4513c79bf39e4e5bfe, server=88aaa70743e7,39293,1733404632706}] 2024-12-05T13:17:20,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure cabc686ba5a0c762c06d5a4d0024dd7b, server=88aaa70743e7,39293,1733404632706}] 2024-12-05T13:17:20,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b3cc71809f62cd16a817f5a172f68ff, server=88aaa70743e7,36055,1733404632807}] 2024-12-05T13:17:20,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01ef88de67b81e433e4128dc0a819605, server=88aaa70743e7,36055,1733404632807}] 2024-12-05T13:17:20,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c03197a9f3300f90c023ccedb4f2112, server=88aaa70743e7,36055,1733404632807}] 2024-12-05T13:17:20,331 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3. 2024-12-05T13:17:20,331 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 97426aa8ec496bfcaaf589a5cbe6f1b3, NAME => 'test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-12-05T13:17:20,331 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:20,331 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:20,331 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:20,331 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:20,333 INFO [StoreOpener-97426aa8ec496bfcaaf589a5cbe6f1b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:20,334 INFO [StoreOpener-97426aa8ec496bfcaaf589a5cbe6f1b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 97426aa8ec496bfcaaf589a5cbe6f1b3 columnFamilyName family 2024-12-05T13:17:20,334 DEBUG [StoreOpener-97426aa8ec496bfcaaf589a5cbe6f1b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:20,334 INFO [StoreOpener-97426aa8ec496bfcaaf589a5cbe6f1b3-1 {}] regionserver.HStore(327): Store=97426aa8ec496bfcaaf589a5cbe6f1b3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:20,334 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:20,335 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:20,337 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:20,338 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:20,338 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for 97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:20,341 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for 97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:20,342 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened 97426aa8ec496bfcaaf589a5cbe6f1b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64776104, jitterRate=-0.03476083278656006}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:20,343 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:20,343 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 97426aa8ec496bfcaaf589a5cbe6f1b3: Running coprocessor pre-open hook at 1733404640331Writing region info on filesystem at 1733404640331Initializing all the Stores at 1733404640332 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404640332Cleaning up temporary data from old regions at 1733404640338 (+6 ms)Running coprocessor post-open hooks at 1733404640343 (+5 ms)Region opened successfully at 1733404640343 2024-12-05T13:17:20,347 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3., pid=74, masterSystemTime=1733404640327 2024-12-05T13:17:20,351 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3. 2024-12-05T13:17:20,351 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3. 2024-12-05T13:17:20,352 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe. 2024-12-05T13:17:20,352 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => eb2ca1f66d6c9b4513c79bf39e4e5bfe, NAME => 'test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-12-05T13:17:20,353 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:20,353 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:20,353 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:20,353 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:20,357 INFO [StoreOpener-eb2ca1f66d6c9b4513c79bf39e4e5bfe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:20,358 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-12-05T13:17:20,358 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,358 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:20,361 INFO [StoreOpener-eb2ca1f66d6c9b4513c79bf39e4e5bfe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb2ca1f66d6c9b4513c79bf39e4e5bfe columnFamilyName family 2024-12-05T13:17:20,361 DEBUG [StoreOpener-eb2ca1f66d6c9b4513c79bf39e4e5bfe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:20,362 INFO [StoreOpener-eb2ca1f66d6c9b4513c79bf39e4e5bfe-1 {}] regionserver.HStore(327): Store=eb2ca1f66d6c9b4513c79bf39e4e5bfe/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:20,362 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:20,369 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:20,371 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:20,371 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:20,371 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:20,375 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:20,374 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-12-05T13:17:20,376 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened eb2ca1f66d6c9b4513c79bf39e4e5bfe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62481864, jitterRate=-0.0689476728439331}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:20,377 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:20,377 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:20,377 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c03197a9f3300f90c023ccedb4f2112, server=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:20,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:20,377 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for eb2ca1f66d6c9b4513c79bf39e4e5bfe: Running coprocessor pre-open hook at 1733404640353Writing region info on filesystem at 1733404640353Initializing all the Stores at 1733404640355 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404640355Cleaning up temporary data from old regions at 1733404640371 (+16 ms)Running coprocessor post-open hooks at 1733404640377 (+6 ms)Region opened successfully at 1733404640377 2024-12-05T13:17:20,378 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c03197a9f3300f90c023ccedb4f2112, server=88aaa70743e7,36055,1733404632807 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:20,383 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe., pid=75, masterSystemTime=1733404640327 2024-12-05T13:17:20,388 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe. 2024-12-05T13:17:20,388 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe. 2024-12-05T13:17:20,388 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07. 2024-12-05T13:17:20,388 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => f8705d72ae6f07bbf8fa8bfbf5a81d07, NAME => 'test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-12-05T13:17:20,388 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:20,388 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:20,388 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:20,389 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:20,390 INFO [StoreOpener-f8705d72ae6f07bbf8fa8bfbf5a81d07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:20,391 INFO [StoreOpener-f8705d72ae6f07bbf8fa8bfbf5a81d07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8705d72ae6f07bbf8fa8bfbf5a81d07 columnFamilyName family 2024-12-05T13:17:20,392 DEBUG [StoreOpener-f8705d72ae6f07bbf8fa8bfbf5a81d07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:20,392 INFO [StoreOpener-f8705d72ae6f07bbf8fa8bfbf5a81d07-1 {}] regionserver.HStore(327): Store=f8705d72ae6f07bbf8fa8bfbf5a81d07/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:20,392 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:20,393 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:20,396 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:20,397 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:20,397 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:20,400 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:20,401 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened f8705d72ae6f07bbf8fa8bfbf5a81d07; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69271627, jitterRate=0.03222768008708954}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:20,401 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:20,401 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for f8705d72ae6f07bbf8fa8bfbf5a81d07: Running coprocessor pre-open hook at 1733404640389Writing region info on filesystem at 1733404640389Initializing all the Stores at 1733404640390 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404640390Cleaning up temporary data from old regions at 1733404640397 (+7 ms)Running coprocessor post-open hooks at 1733404640401 (+4 ms)Region opened successfully at 1733404640401 2024-12-05T13:17:20,402 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07., pid=72, masterSystemTime=1733404640327 2024-12-05T13:17:20,405 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07. 2024-12-05T13:17:20,405 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07. 2024-12-05T13:17:20,405 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b. 2024-12-05T13:17:20,406 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => cabc686ba5a0c762c06d5a4d0024dd7b, NAME => 'test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-12-05T13:17:20,406 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:20,406 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:20,406 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:20,406 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:20,409 INFO [StoreOpener-cabc686ba5a0c762c06d5a4d0024dd7b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:20,411 INFO [StoreOpener-cabc686ba5a0c762c06d5a4d0024dd7b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cabc686ba5a0c762c06d5a4d0024dd7b columnFamilyName family 2024-12-05T13:17:20,411 DEBUG [StoreOpener-cabc686ba5a0c762c06d5a4d0024dd7b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:20,412 INFO [StoreOpener-cabc686ba5a0c762c06d5a4d0024dd7b-1 {}] regionserver.HStore(327): Store=cabc686ba5a0c762c06d5a4d0024dd7b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:20,412 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:20,413 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:20,415 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:20,417 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:20,417 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:20,420 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:20,421 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened cabc686ba5a0c762c06d5a4d0024dd7b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71911186, jitterRate=0.07156017422676086}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:20,421 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:20,421 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for cabc686ba5a0c762c06d5a4d0024dd7b: Running coprocessor pre-open hook at 1733404640406Writing region info on filesystem at 1733404640406Initializing all the Stores at 1733404640408 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404640408Cleaning up temporary data from old regions at 1733404640417 (+9 ms)Running coprocessor post-open hooks at 1733404640421 (+4 ms)Region opened successfully at 1733404640421 2024-12-05T13:17:20,423 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b., pid=76, masterSystemTime=1733404640327 2024-12-05T13:17:20,425 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b. 2024-12-05T13:17:20,426 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b. 2024-12-05T13:17:20,436 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5. 2024-12-05T13:17:20,436 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => cb80ccb4b3541210c6fda8b4b988adb5, NAME => 'test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-12-05T13:17:20,437 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:20,437 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:20,437 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:20,437 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:20,439 INFO [StoreOpener-cb80ccb4b3541210c6fda8b4b988adb5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:20,440 INFO [StoreOpener-cb80ccb4b3541210c6fda8b4b988adb5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb80ccb4b3541210c6fda8b4b988adb5 columnFamilyName family 2024-12-05T13:17:20,440 DEBUG [StoreOpener-cb80ccb4b3541210c6fda8b4b988adb5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:20,441 INFO [StoreOpener-cb80ccb4b3541210c6fda8b4b988adb5-1 {}] regionserver.HStore(327): Store=cb80ccb4b3541210c6fda8b4b988adb5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:20,441 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:20,444 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:20,446 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:20,446 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:20,446 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:20,449 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:20,450 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened cb80ccb4b3541210c6fda8b4b988adb5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72916969, jitterRate=0.08654750883579254}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:20,451 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:20,451 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for cb80ccb4b3541210c6fda8b4b988adb5: Running coprocessor pre-open hook at 1733404640437Writing region info on filesystem at 1733404640437Initializing all the Stores at 1733404640438 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404640438Cleaning up temporary data from old regions at 1733404640446 (+8 ms)Running coprocessor post-open hooks at 1733404640451 (+5 ms)Region opened successfully at 1733404640451 2024-12-05T13:17:20,452 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5., pid=73, masterSystemTime=1733404640431 2024-12-05T13:17:20,454 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5. 2024-12-05T13:17:20,454 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5. 2024-12-05T13:17:20,454 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,iii,1733404635844.01ef88de67b81e433e4128dc0a819605. 2024-12-05T13:17:20,455 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 01ef88de67b81e433e4128dc0a819605, NAME => 'test,iii,1733404635844.01ef88de67b81e433e4128dc0a819605.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-12-05T13:17:20,455 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 01ef88de67b81e433e4128dc0a819605 2024-12-05T13:17:20,455 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,iii,1733404635844.01ef88de67b81e433e4128dc0a819605.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T13:17:20,455 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 01ef88de67b81e433e4128dc0a819605 2024-12-05T13:17:20,455 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 01ef88de67b81e433e4128dc0a819605 2024-12-05T13:17:20,457 INFO [StoreOpener-01ef88de67b81e433e4128dc0a819605-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 01ef88de67b81e433e4128dc0a819605 2024-12-05T13:17:20,459 INFO [StoreOpener-01ef88de67b81e433e4128dc0a819605-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01ef88de67b81e433e4128dc0a819605 columnFamilyName family 2024-12-05T13:17:20,459 DEBUG [StoreOpener-01ef88de67b81e433e4128dc0a819605-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T13:17:20,459 INFO [StoreOpener-01ef88de67b81e433e4128dc0a819605-1 {}] regionserver.HStore(327): Store=01ef88de67b81e433e4128dc0a819605/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T13:17:20,459 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 01ef88de67b81e433e4128dc0a819605 2024-12-05T13:17:20,461 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/01ef88de67b81e433e4128dc0a819605 2024-12-05T13:17:20,463 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:20,463 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:20,463 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:20,463 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:20,464 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/01ef88de67b81e433e4128dc0a819605 2024-12-05T13:17:20,464 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:20,464 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 01ef88de67b81e433e4128dc0a819605 2024-12-05T13:17:20,464 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:20,464 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:20,464 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 01ef88de67b81e433e4128dc0a819605 2024-12-05T13:17:20,465 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:20,466 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-05T13:17:20,469 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 01ef88de67b81e433e4128dc0a819605 2024-12-05T13:17:20,471 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 01ef88de67b81e433e4128dc0a819605; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65611330, jitterRate=-0.022314995527267456}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T13:17:20,471 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 01ef88de67b81e433e4128dc0a819605 2024-12-05T13:17:20,472 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 01ef88de67b81e433e4128dc0a819605: Running coprocessor pre-open hook at 1733404640455Writing region info on filesystem at 1733404640455Initializing all the Stores at 1733404640457 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733404640457Cleaning up temporary data from old regions at 1733404640464 (+7 ms)Running coprocessor post-open hooks at 1733404640472 (+8 ms)Region opened successfully at 1733404640472 2024-12-05T13:17:20,473 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1733404635844.01ef88de67b81e433e4128dc0a819605., pid=78, masterSystemTime=1733404640431 2024-12-05T13:17:20,474 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:20,475 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,474 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:20,475 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,475 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,475 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,475 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,475 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,475 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,475 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,475 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,475 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,475 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,475 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,475 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,475 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,475 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,475 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,476 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:20,476 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:20,476 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,476 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,476 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:20,476 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:20,477 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:20,476 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "88aaa70743e7" port: 36055 start_code: 1733404632807 } transition { transition_code: OPENED region_info { region_id: 1733404635844 table_name { namespace: "default" qualifier: "test" } start_key: "iii" end_key: "jjj" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 78 initiating_master_active_time: 1733404632952 }; retry (#0) after 1002ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-05T13:17:20,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:20,487 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:20,487 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,487 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8705d72ae6f07bbf8fa8bfbf5a81d07, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb80ccb4b3541210c6fda8b4b988adb5, server=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb2ca1f66d6c9b4513c79bf39e4e5bfe, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure cabc686ba5a0c762c06d5a4d0024dd7b, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b3cc71809f62cd16a817f5a172f68ff, server=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01ef88de67b81e433e4128dc0a819605, server=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c03197a9f3300f90c023ccedb4f2112, server=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,488 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,488 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:20,489 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,489 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:20,489 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,489 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:20,489 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,489 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:20,489 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,489 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:20,489 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,489 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:20,489 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,489 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:20,489 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,489 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN 2024-12-05T13:17:20,489 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8705d72ae6f07bbf8fa8bfbf5a81d07, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,489 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb80ccb4b3541210c6fda8b4b988adb5, server=88aaa70743e7,36055,1733404632807 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,489 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,489 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb2ca1f66d6c9b4513c79bf39e4e5bfe, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,489 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure cabc686ba5a0c762c06d5a4d0024dd7b, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,489 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b3cc71809f62cd16a817f5a172f68ff, server=88aaa70743e7,36055,1733404632807 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,489 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01ef88de67b81e433e4128dc0a819605, server=88aaa70743e7,36055,1733404632807 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,489 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c03197a9f3300f90c023ccedb4f2112, server=88aaa70743e7,36055,1733404632807 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,490 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 23 msec 2024-12-05T13:17:20,491 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:20,491 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:20,492 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:20,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:20,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:20,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:20,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:20,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:20,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:20,501 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN to state=OFFLINE, location=null, table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3 to restore RIT 2024-12-05T13:17:20,501 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN to state=OFFLINE, location=null, table=test, region=cb80ccb4b3541210c6fda8b4b988adb5 to restore RIT 2024-12-05T13:17:20,501 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN to state=OFFLINE, location=null, table=test, region=3c03197a9f3300f90c023ccedb4f2112 to restore RIT 2024-12-05T13:17:20,501 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN to state=OFFLINE, location=null, table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe to restore RIT 2024-12-05T13:17:20,501 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN to state=OFFLINE, location=null, table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b to restore RIT 2024-12-05T13:17:20,501 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN to state=OFFLINE, location=null, table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07 to restore RIT 2024-12-05T13:17:20,501 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN to state=OFFLINE, location=null, table=test, region=8b3cc71809f62cd16a817f5a172f68ff to restore RIT 2024-12-05T13:17:20,502 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN to state=OFFLINE, location=null, table=test, region=01ef88de67b81e433e4128dc0a819605 to restore RIT 2024-12-05T13:17:20,502 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404670502 2024-12-05T13:17:20,502 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:20,502 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:20,504 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:20,505 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:20,506 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01ef88de67b81e433e4128dc0a819605, server=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:20,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:20,506 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01ef88de67b81e433e4128dc0a819605, server=88aaa70743e7,36055,1733404632807 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:20,535 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,536 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,537 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,538 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,539 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,540 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,542 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,542 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:20,542 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404700542 2024-12-05T13:17:20,542 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404760542 2024-12-05T13:17:20,543 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 40 msec 2024-12-05T13:17:20,543 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:20,543 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:20,543 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:20,543 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:20,543 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:20,543 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:20,543 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:20,544 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:20,545 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T13:17:20,545 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:20,550 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:20,550 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,550 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,550 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,550 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:20,550 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,550 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,550 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:20,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:20,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:20,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:20,552 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:20,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:20,556 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:20,556 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,556 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,556 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:20,556 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,556 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:20,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:20,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:20,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:20,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:20,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:20,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:20,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN 2024-12-05T13:17:20,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8705d72ae6f07bbf8fa8bfbf5a81d07, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb80ccb4b3541210c6fda8b4b988adb5, server=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:20,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb2ca1f66d6c9b4513c79bf39e4e5bfe, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure cabc686ba5a0c762c06d5a4d0024dd7b, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b3cc71809f62cd16a817f5a172f68ff, server=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:20,557 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01ef88de67b81e433e4128dc0a819605, server=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:20,558 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,558 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,558 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,558 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,558 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,558 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,558 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,558 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,558 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:20,558 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,558 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:20,558 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,558 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:20,558 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,558 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:20,558 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,558 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:20,558 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,558 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:20,559 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,559 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:20,559 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,559 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN 2024-12-05T13:17:20,559 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8705d72ae6f07bbf8fa8bfbf5a81d07, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,559 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb80ccb4b3541210c6fda8b4b988adb5, server=88aaa70743e7,36055,1733404632807 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,559 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,559 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb2ca1f66d6c9b4513c79bf39e4e5bfe, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,559 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure cabc686ba5a0c762c06d5a4d0024dd7b, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,559 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b3cc71809f62cd16a817f5a172f68ff, server=88aaa70743e7,36055,1733404632807 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,559 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01ef88de67b81e433e4128dc0a819605, server=88aaa70743e7,36055,1733404632807 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,559 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-12-05T13:17:20,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:20,561 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:20,561 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:20,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:20,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:20,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:20,578 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:20,578 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:20,579 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:20,579 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN to state=OFFLINE, location=null, table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3 to restore RIT 2024-12-05T13:17:20,579 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN to state=OFFLINE, location=null, table=test, region=cb80ccb4b3541210c6fda8b4b988adb5 to restore RIT 2024-12-05T13:17:20,579 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN to state=OFFLINE, location=null, table=test, region=3c03197a9f3300f90c023ccedb4f2112 to restore RIT 2024-12-05T13:17:20,579 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN to state=OFFLINE, location=null, table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe to restore RIT 2024-12-05T13:17:20,579 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN to state=OFFLINE, location=null, table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b to restore RIT 2024-12-05T13:17:20,580 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN to state=OFFLINE, location=null, table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07 to restore RIT 2024-12-05T13:17:20,580 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN to state=OFFLINE, location=null, table=test, region=8b3cc71809f62cd16a817f5a172f68ff to restore RIT 2024-12-05T13:17:20,580 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN to state=OFFLINE, location=null, table=test, region=01ef88de67b81e433e4128dc0a819605 to restore RIT 2024-12-05T13:17:20,580 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404670580 2024-12-05T13:17:20,580 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:20,580 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:20,582 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:20,587 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:20,587 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b3cc71809f62cd16a817f5a172f68ff, server=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:20,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:20,587 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b3cc71809f62cd16a817f5a172f68ff, server=88aaa70743e7,36055,1733404632807 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:20,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,594 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:20,594 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404700594 2024-12-05T13:17:20,594 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404760594 2024-12-05T13:17:20,594 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-12-05T13:17:20,594 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:20,594 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:20,594 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:20,594 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:20,595 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:20,595 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:20,595 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:20,595 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:20,596 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T13:17:20,601 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:20,601 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,601 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,601 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,601 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,601 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,601 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,601 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:20,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:20,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:20,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:20,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:20,604 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:20,607 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:20,607 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,607 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:20,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:20,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:20,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:20,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:20,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:20,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:20,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:20,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN 2024-12-05T13:17:20,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8705d72ae6f07bbf8fa8bfbf5a81d07, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb80ccb4b3541210c6fda8b4b988adb5, server=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:20,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb2ca1f66d6c9b4513c79bf39e4e5bfe, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure cabc686ba5a0c762c06d5a4d0024dd7b, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,609 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b3cc71809f62cd16a817f5a172f68ff, server=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:20,609 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,609 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,609 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,609 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,609 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,609 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,609 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,609 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,609 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:20,609 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,609 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:20,609 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,609 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:20,609 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,609 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:20,609 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,609 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:20,609 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,609 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:20,609 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,609 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:20,609 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,609 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN 2024-12-05T13:17:20,609 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8705d72ae6f07bbf8fa8bfbf5a81d07, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,609 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb80ccb4b3541210c6fda8b4b988adb5, server=88aaa70743e7,36055,1733404632807 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,609 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,609 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb2ca1f66d6c9b4513c79bf39e4e5bfe, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,609 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure cabc686ba5a0c762c06d5a4d0024dd7b, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,609 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b3cc71809f62cd16a817f5a172f68ff, server=88aaa70743e7,36055,1733404632807 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,610 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-12-05T13:17:20,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:20,611 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:20,611 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:20,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:20,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:20,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:20,628 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:20,628 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:20,628 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:20,629 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN to state=OFFLINE, location=null, table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3 to restore RIT 2024-12-05T13:17:20,629 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN to state=OFFLINE, location=null, table=test, region=cb80ccb4b3541210c6fda8b4b988adb5 to restore RIT 2024-12-05T13:17:20,629 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN to state=OFFLINE, location=null, table=test, region=3c03197a9f3300f90c023ccedb4f2112 to restore RIT 2024-12-05T13:17:20,629 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN to state=OFFLINE, location=null, table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe to restore RIT 2024-12-05T13:17:20,629 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN to state=OFFLINE, location=null, table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b to restore RIT 2024-12-05T13:17:20,629 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN to state=OFFLINE, location=null, table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07 to restore RIT 2024-12-05T13:17:20,629 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN to state=OFFLINE, location=null, table=test, region=8b3cc71809f62cd16a817f5a172f68ff to restore RIT 2024-12-05T13:17:20,629 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN to state=OFFLINE, location=null, table=test, region=01ef88de67b81e433e4128dc0a819605 to restore RIT 2024-12-05T13:17:20,630 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404670630 2024-12-05T13:17:20,630 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:20,630 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:20,632 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:20,633 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:20,633 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure cabc686ba5a0c762c06d5a4d0024dd7b, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:20,633 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure cabc686ba5a0c762c06d5a4d0024dd7b, server=88aaa70743e7,39293,1733404632706 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:20,638 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,638 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,638 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,638 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,639 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,640 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,641 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,641 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:20,641 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404700641 2024-12-05T13:17:20,641 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404760641 2024-12-05T13:17:20,642 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-12-05T13:17:20,642 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:20,642 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:20,642 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:20,642 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:20,642 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:20,642 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:20,642 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:20,642 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:20,643 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T13:17:20,646 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:20,646 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,646 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,646 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,646 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,646 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,646 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,646 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,646 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,646 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,646 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,646 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,646 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,646 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,647 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:20,647 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:20,647 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,647 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,647 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:20,647 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:20,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:20,648 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:20,649 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:20,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:20,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:20,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:20,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:20,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:20,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:20,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:20,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:20,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:20,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN 2024-12-05T13:17:20,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8705d72ae6f07bbf8fa8bfbf5a81d07, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,650 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb80ccb4b3541210c6fda8b4b988adb5, server=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:20,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb2ca1f66d6c9b4513c79bf39e4e5bfe, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure cabc686ba5a0c762c06d5a4d0024dd7b, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,651 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,651 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,651 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,651 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,651 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,651 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,651 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,651 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,651 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:20,651 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,651 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:20,651 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,651 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:20,651 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,651 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:20,651 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,651 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:20,651 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,651 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:20,651 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,651 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:20,651 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,651 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN 2024-12-05T13:17:20,651 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8705d72ae6f07bbf8fa8bfbf5a81d07, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,651 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb80ccb4b3541210c6fda8b4b988adb5, server=88aaa70743e7,36055,1733404632807 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,651 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,651 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb2ca1f66d6c9b4513c79bf39e4e5bfe, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,652 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure cabc686ba5a0c762c06d5a4d0024dd7b, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,652 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-12-05T13:17:20,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:20,653 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:20,653 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:20,654 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:20,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:20,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:20,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:20,670 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:20,670 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:20,670 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:20,671 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN to state=OFFLINE, location=null, table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3 to restore RIT 2024-12-05T13:17:20,671 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN to state=OFFLINE, location=null, table=test, region=cb80ccb4b3541210c6fda8b4b988adb5 to restore RIT 2024-12-05T13:17:20,671 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN to state=OFFLINE, location=null, table=test, region=3c03197a9f3300f90c023ccedb4f2112 to restore RIT 2024-12-05T13:17:20,671 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN to state=OFFLINE, location=null, table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe to restore RIT 2024-12-05T13:17:20,671 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN to state=OFFLINE, location=null, table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b to restore RIT 2024-12-05T13:17:20,671 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN to state=OFFLINE, location=null, table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07 to restore RIT 2024-12-05T13:17:20,671 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN to state=OFFLINE, location=null, table=test, region=8b3cc71809f62cd16a817f5a172f68ff to restore RIT 2024-12-05T13:17:20,671 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN to state=OFFLINE, location=null, table=test, region=01ef88de67b81e433e4128dc0a819605 to restore RIT 2024-12-05T13:17:20,672 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404670672 2024-12-05T13:17:20,672 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:20,672 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:20,674 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:20,675 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:20,675 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb2ca1f66d6c9b4513c79bf39e4e5bfe, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:20,675 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb2ca1f66d6c9b4513c79bf39e4e5bfe, server=88aaa70743e7,39293,1733404632706 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:20,680 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,681 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,683 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,684 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,685 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,686 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,686 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:20,686 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404700686 2024-12-05T13:17:20,686 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404760686 2024-12-05T13:17:20,687 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-12-05T13:17:20,687 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:20,687 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:20,687 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:20,687 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:20,687 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:20,687 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:20,687 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:20,687 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:20,688 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T13:17:20,691 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:20,691 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,691 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,691 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,691 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:20,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:20,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:20,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:20,692 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:20,693 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:20,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:20,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:20,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:20,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:20,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:20,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:20,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:20,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:20,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:20,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN 2024-12-05T13:17:20,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8705d72ae6f07bbf8fa8bfbf5a81d07, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb80ccb4b3541210c6fda8b4b988adb5, server=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:20,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb2ca1f66d6c9b4513c79bf39e4e5bfe, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,695 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,695 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,696 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,696 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,696 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,696 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,696 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,696 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,696 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:20,696 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,696 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:20,696 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,696 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:20,696 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,696 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:20,696 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,696 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:20,696 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,696 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:20,696 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,696 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:20,696 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,697 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN 2024-12-05T13:17:20,697 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8705d72ae6f07bbf8fa8bfbf5a81d07, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,697 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb80ccb4b3541210c6fda8b4b988adb5, server=88aaa70743e7,36055,1733404632807 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,697 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,697 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb2ca1f66d6c9b4513c79bf39e4e5bfe, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,697 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-12-05T13:17:20,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:20,698 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:20,699 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:20,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:20,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:20,708 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:20,708 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:20,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:20,708 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:20,709 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN to state=OFFLINE, location=null, table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3 to restore RIT 2024-12-05T13:17:20,709 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN to state=OFFLINE, location=null, table=test, region=cb80ccb4b3541210c6fda8b4b988adb5 to restore RIT 2024-12-05T13:17:20,709 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN to state=OFFLINE, location=null, table=test, region=3c03197a9f3300f90c023ccedb4f2112 to restore RIT 2024-12-05T13:17:20,709 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN to state=OFFLINE, location=null, table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe to restore RIT 2024-12-05T13:17:20,709 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN to state=OFFLINE, location=null, table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b to restore RIT 2024-12-05T13:17:20,709 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN to state=OFFLINE, location=null, table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07 to restore RIT 2024-12-05T13:17:20,709 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN to state=OFFLINE, location=null, table=test, region=8b3cc71809f62cd16a817f5a172f68ff to restore RIT 2024-12-05T13:17:20,709 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN to state=OFFLINE, location=null, table=test, region=01ef88de67b81e433e4128dc0a819605 to restore RIT 2024-12-05T13:17:20,710 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404670710 2024-12-05T13:17:20,710 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:20,710 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:20,712 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:20,714 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:20,714 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:20,715 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:20,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,721 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,722 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,723 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,723 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:20,723 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404700723 2024-12-05T13:17:20,723 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404760723 2024-12-05T13:17:20,723 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-12-05T13:17:20,723 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:20,723 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:20,723 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:20,724 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:20,724 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:20,724 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:20,724 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:20,724 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:20,725 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T13:17:20,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:20,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,728 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:20,729 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:20,729 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:20,729 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:20,729 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:20,729 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:20,729 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:20,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:20,733 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:20,733 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,733 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,733 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:20,733 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,733 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,733 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,733 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:20,733 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:20,733 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:20,733 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:20,733 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:20,733 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:20,733 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:20,733 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN 2024-12-05T13:17:20,733 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8705d72ae6f07bbf8fa8bfbf5a81d07, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,734 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb80ccb4b3541210c6fda8b4b988adb5, server=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:20,734 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:20,734 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,734 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,734 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,734 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,734 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,734 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,734 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,734 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,734 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:20,734 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,734 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:20,734 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,734 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:20,734 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,734 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:20,734 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,734 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:20,735 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,735 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:20,735 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,735 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:20,735 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:20,735 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN 2024-12-05T13:17:20,735 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8705d72ae6f07bbf8fa8bfbf5a81d07, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,735 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb80ccb4b3541210c6fda8b4b988adb5, server=88aaa70743e7,36055,1733404632807 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,735 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:20,735 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-12-05T13:17:20,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:20,736 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:20,736 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:20,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:20,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:20,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:20,745 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:20,745 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:20,745 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:20,746 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN to state=OFFLINE, location=null, table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3 to restore RIT 2024-12-05T13:17:20,746 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN to state=OFFLINE, location=null, table=test, region=cb80ccb4b3541210c6fda8b4b988adb5 to restore RIT 2024-12-05T13:17:20,746 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN to state=OFFLINE, location=null, table=test, region=3c03197a9f3300f90c023ccedb4f2112 to restore RIT 2024-12-05T13:17:20,746 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN to state=OFFLINE, location=null, table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe to restore RIT 2024-12-05T13:17:20,746 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN to state=OFFLINE, location=null, table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b to restore RIT 2024-12-05T13:17:20,746 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN to state=OFFLINE, location=null, table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07 to restore RIT 2024-12-05T13:17:20,746 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN to state=OFFLINE, location=null, table=test, region=8b3cc71809f62cd16a817f5a172f68ff to restore RIT 2024-12-05T13:17:20,746 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN to state=OFFLINE, location=null, table=test, region=01ef88de67b81e433e4128dc0a819605 to restore RIT 2024-12-05T13:17:20,747 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404670747 2024-12-05T13:17:20,747 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:20,747 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:20,755 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:20,757 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,758 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,759 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,760 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:20,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:20,762 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:20,762 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404700762 2024-12-05T13:17:20,762 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404760762 2024-12-05T13:17:20,763 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-12-05T13:17:20,765 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:20,765 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb80ccb4b3541210c6fda8b4b988adb5, server=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:20,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:20,766 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb80ccb4b3541210c6fda8b4b988adb5, server=88aaa70743e7,36055,1733404632807 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:20,783 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T13:17:20,841 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-12-05T13:17:21,013 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:21,013 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,013 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:21,013 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:21,013 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:21,013 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:21,013 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:21,014 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:21,015 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T13:17:21,017 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:21,017 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,017 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,017 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:21,017 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,017 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:21,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:21,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,018 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,018 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:21,019 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:21,021 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:21,021 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,021 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:21,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:21,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:21,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:21,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:21,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:21,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:21,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:21,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN 2024-12-05T13:17:21,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8705d72ae6f07bbf8fa8bfbf5a81d07, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,022 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb80ccb4b3541210c6fda8b4b988adb5, server=88aaa70743e7,36055,1733404632807 2024-12-05T13:17:21,022 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,022 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,022 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,022 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,022 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,022 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,022 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,022 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,022 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:21,023 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,023 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:21,023 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,023 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:21,023 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,023 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:21,023 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,023 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:21,023 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,023 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:21,023 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,023 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:21,023 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,023 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN 2024-12-05T13:17:21,023 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8705d72ae6f07bbf8fa8bfbf5a81d07, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,023 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb80ccb4b3541210c6fda8b4b988adb5, server=88aaa70743e7,36055,1733404632807 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,023 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-12-05T13:17:21,024 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:21,024 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:21,024 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:21,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,033 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,033 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,033 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,034 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN to state=OFFLINE, location=null, table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3 to restore RIT 2024-12-05T13:17:21,034 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN to state=OFFLINE, location=null, table=test, region=cb80ccb4b3541210c6fda8b4b988adb5 to restore RIT 2024-12-05T13:17:21,034 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN to state=OFFLINE, location=null, table=test, region=3c03197a9f3300f90c023ccedb4f2112 to restore RIT 2024-12-05T13:17:21,034 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN to state=OFFLINE, location=null, table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe to restore RIT 2024-12-05T13:17:21,034 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN to state=OFFLINE, location=null, table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b to restore RIT 2024-12-05T13:17:21,034 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN to state=OFFLINE, location=null, table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07 to restore RIT 2024-12-05T13:17:21,034 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN to state=OFFLINE, location=null, table=test, region=8b3cc71809f62cd16a817f5a172f68ff to restore RIT 2024-12-05T13:17:21,034 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN to state=OFFLINE, location=null, table=test, region=01ef88de67b81e433e4128dc0a819605 to restore RIT 2024-12-05T13:17:21,035 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404671035 2024-12-05T13:17:21,035 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:21,035 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,037 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:21,043 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:21,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,043 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8705d72ae6f07bbf8fa8bfbf5a81d07, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,043 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,043 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8705d72ae6f07bbf8fa8bfbf5a81d07, server=88aaa70743e7,39293,1733404632706 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,044 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,045 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,046 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,047 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,048 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,048 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:21,048 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404701048 2024-12-05T13:17:21,048 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404761048 2024-12-05T13:17:21,048 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-12-05T13:17:21,048 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:21,048 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,048 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:21,049 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:21,049 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:21,049 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:21,049 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:21,050 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:21,051 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-05T13:17:21,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:21,053 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:21,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:21,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,054 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,055 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:21,057 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8705d72ae6f07bbf8fa8bfbf5a81d07, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,060 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,061 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,061 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:21,061 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,061 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:21,061 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,061 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:21,061 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,061 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:21,061 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,061 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:21,061 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,061 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:21,061 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,061 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:21,061 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,061 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN 2024-12-05T13:17:21,061 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8705d72ae6f07bbf8fa8bfbf5a81d07, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,061 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-12-05T13:17:21,063 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:21,063 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:21,063 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:21,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,070 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,070 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,070 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,072 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN to state=OFFLINE, location=null, table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3 to restore RIT 2024-12-05T13:17:21,072 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN to state=OFFLINE, location=null, table=test, region=cb80ccb4b3541210c6fda8b4b988adb5 to restore RIT 2024-12-05T13:17:21,072 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN to state=OFFLINE, location=null, table=test, region=3c03197a9f3300f90c023ccedb4f2112 to restore RIT 2024-12-05T13:17:21,072 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN to state=OFFLINE, location=null, table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe to restore RIT 2024-12-05T13:17:21,072 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN to state=OFFLINE, location=null, table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b to restore RIT 2024-12-05T13:17:21,072 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN to state=OFFLINE, location=null, table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07 to restore RIT 2024-12-05T13:17:21,072 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN to state=OFFLINE, location=null, table=test, region=8b3cc71809f62cd16a817f5a172f68ff to restore RIT 2024-12-05T13:17:21,072 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN to state=OFFLINE, location=null, table=test, region=01ef88de67b81e433e4128dc0a819605 to restore RIT 2024-12-05T13:17:21,074 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404671074 2024-12-05T13:17:21,074 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:21,074 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,075 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:21,076 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:21,076 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN 2024-12-05T13:17:21,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,077 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,082 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,083 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,083 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:21,083 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404701083 2024-12-05T13:17:21,083 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404761083 2024-12-05T13:17:21,083 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-05T13:17:21,083 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:21,083 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,083 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:21,084 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:21,084 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:21,084 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:21,084 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:21,084 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:21,085 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T13:17:21,086 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:21,086 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,086 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,086 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,086 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,086 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,086 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,086 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,086 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,086 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,086 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,086 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,086 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:21,086 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,087 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:21,087 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:21,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:21,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:21,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,088 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:21,089 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:21,089 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:21,089 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:21,089 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:21,089 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:21,089 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:21,089 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN 2024-12-05T13:17:21,089 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,089 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,089 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,089 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,089 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,089 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,089 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,089 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,089 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:21,089 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,089 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:21,089 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,089 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:21,089 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,089 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:21,089 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,089 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:21,089 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,089 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:21,089 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,089 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:21,089 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,089 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN 2024-12-05T13:17:21,089 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-05T13:17:21,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:21,090 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:21,090 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:21,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,099 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,099 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,099 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN to state=OFFLINE, location=null, table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3 to restore RIT 2024-12-05T13:17:21,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN to state=OFFLINE, location=null, table=test, region=cb80ccb4b3541210c6fda8b4b988adb5 to restore RIT 2024-12-05T13:17:21,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN to state=OFFLINE, location=null, table=test, region=3c03197a9f3300f90c023ccedb4f2112 to restore RIT 2024-12-05T13:17:21,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN to state=OFFLINE, location=null, table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe to restore RIT 2024-12-05T13:17:21,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN to state=OFFLINE, location=null, table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b to restore RIT 2024-12-05T13:17:21,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN to state=OFFLINE, location=null, table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07 to restore RIT 2024-12-05T13:17:21,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN to state=OFFLINE, location=null, table=test, region=8b3cc71809f62cd16a817f5a172f68ff to restore RIT 2024-12-05T13:17:21,100 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=01ef88de67b81e433e4128dc0a819605, ASSIGN to state=OFFLINE, location=null, table=test, region=01ef88de67b81e433e4128dc0a819605 to restore RIT 2024-12-05T13:17:21,101 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404671101 2024-12-05T13:17:21,101 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:21,101 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,102 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:21,106 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,107 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:21,107 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:21,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,107 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,108 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,109 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,110 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,110 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:21,110 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404701110 2024-12-05T13:17:21,110 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404761110 2024-12-05T13:17:21,110 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-05T13:17:21,111 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:21,111 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,111 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:21,111 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:21,111 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:21,111 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:21,111 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:21,111 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:21,112 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T13:17:21,114 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:21,114 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:21,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,115 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,115 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:21,116 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:21,117 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:21,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:21,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:21,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:21,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:21,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:21,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:21,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:21,118 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:21,118 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,118 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,118 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,118 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,118 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,118 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,118 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,118 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,119 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:21,119 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,119 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:21,119 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,119 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:21,119 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,119 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:21,119 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,119 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:21,119 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,119 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:21,119 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,119 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN 2024-12-05T13:17:21,119 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-05T13:17:21,120 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:21,120 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:21,121 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:21,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,128 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,128 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,128 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,129 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN to state=OFFLINE, location=null, table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3 to restore RIT 2024-12-05T13:17:21,129 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN to state=OFFLINE, location=null, table=test, region=cb80ccb4b3541210c6fda8b4b988adb5 to restore RIT 2024-12-05T13:17:21,129 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN to state=OFFLINE, location=null, table=test, region=3c03197a9f3300f90c023ccedb4f2112 to restore RIT 2024-12-05T13:17:21,129 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN to state=OFFLINE, location=null, table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe to restore RIT 2024-12-05T13:17:21,129 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN to state=OFFLINE, location=null, table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b to restore RIT 2024-12-05T13:17:21,129 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN to state=OFFLINE, location=null, table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07 to restore RIT 2024-12-05T13:17:21,129 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8b3cc71809f62cd16a817f5a172f68ff, ASSIGN to state=OFFLINE, location=null, table=test, region=8b3cc71809f62cd16a817f5a172f68ff to restore RIT 2024-12-05T13:17:21,131 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404671131 2024-12-05T13:17:21,131 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:21,131 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,133 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:21,137 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,137 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,137 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,137 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:21,138 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:21,138 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,138 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,138 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,138 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,138 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,138 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,138 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,138 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,139 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,139 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,139 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,139 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,139 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,139 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,139 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,139 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,139 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,140 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,140 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,140 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,140 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,140 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,140 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,140 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,140 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:21,140 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404701140 2024-12-05T13:17:21,140 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404761140 2024-12-05T13:17:21,140 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-05T13:17:21,141 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:21,141 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,141 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:21,141 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:21,141 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:21,141 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:21,141 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:21,141 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:21,142 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T13:17:21,143 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:21,143 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,143 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,143 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,143 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,143 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,143 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:21,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,144 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,144 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:21,144 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:21,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:21,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:21,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,145 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,146 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:21,146 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:21,146 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:21,146 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:21,146 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:21,146 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:21,146 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,146 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,146 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,146 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:21,146 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,146 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:21,146 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,146 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:21,146 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,146 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:21,146 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,146 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:21,147 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,147 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN 2024-12-05T13:17:21,147 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-05T13:17:21,148 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:21,148 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:21,148 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:21,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,162 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,162 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,162 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,162 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN to state=OFFLINE, location=null, table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3 to restore RIT 2024-12-05T13:17:21,163 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN to state=OFFLINE, location=null, table=test, region=cb80ccb4b3541210c6fda8b4b988adb5 to restore RIT 2024-12-05T13:17:21,163 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN to state=OFFLINE, location=null, table=test, region=3c03197a9f3300f90c023ccedb4f2112 to restore RIT 2024-12-05T13:17:21,163 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN to state=OFFLINE, location=null, table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe to restore RIT 2024-12-05T13:17:21,163 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN to state=OFFLINE, location=null, table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b to restore RIT 2024-12-05T13:17:21,163 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07, ASSIGN to state=OFFLINE, location=null, table=test, region=f8705d72ae6f07bbf8fa8bfbf5a81d07 to restore RIT 2024-12-05T13:17:21,163 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404671163 2024-12-05T13:17:21,163 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:21,163 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,165 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:21,168 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:21,168 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:21,168 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,169 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,172 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:21,172 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404701172 2024-12-05T13:17:21,172 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404761172 2024-12-05T13:17:21,172 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-05T13:17:21,172 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:21,172 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,172 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:21,172 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:21,172 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:21,172 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:21,172 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:21,173 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:21,175 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-12-05T13:17:21,175 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,177 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:21,177 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,177 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,178 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,178 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,178 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,178 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,178 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,178 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,178 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:21,178 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,178 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,178 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:21,179 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:21,181 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:21,181 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,181 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,181 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:21,181 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,181 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,181 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,181 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:21,181 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:21,181 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:21,181 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:21,182 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:21,182 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,182 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,182 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,182 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,182 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,182 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,182 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,182 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,182 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:21,182 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,182 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:21,182 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,182 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:21,182 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,182 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:21,182 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,182 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN 2024-12-05T13:17:21,182 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-05T13:17:21,183 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:21,183 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:21,184 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:21,184 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,191 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,191 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,191 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,191 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,193 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN to state=OFFLINE, location=null, table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3 to restore RIT 2024-12-05T13:17:21,193 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN to state=OFFLINE, location=null, table=test, region=cb80ccb4b3541210c6fda8b4b988adb5 to restore RIT 2024-12-05T13:17:21,193 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN to state=OFFLINE, location=null, table=test, region=3c03197a9f3300f90c023ccedb4f2112 to restore RIT 2024-12-05T13:17:21,193 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN to state=OFFLINE, location=null, table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe to restore RIT 2024-12-05T13:17:21,193 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b, ASSIGN to state=OFFLINE, location=null, table=test, region=cabc686ba5a0c762c06d5a4d0024dd7b to restore RIT 2024-12-05T13:17:21,193 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404671193 2024-12-05T13:17:21,194 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:21,194 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,195 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:21,197 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:21,197 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:21,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,197 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,200 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,203 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:21,204 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404701203 2024-12-05T13:17:21,204 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404761204 2024-12-05T13:17:21,204 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-05T13:17:21,204 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:21,204 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,204 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:21,204 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:21,204 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:21,204 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:21,204 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:21,205 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:21,205 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T13:17:21,207 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:21,207 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,208 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,208 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,208 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,208 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,208 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,208 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,208 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:21,208 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,208 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:21,209 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:21,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:21,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:21,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:21,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:21,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:21,210 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:21,210 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,210 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,210 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,210 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,210 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,210 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,210 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,210 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,210 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:21,211 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,211 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:21,211 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,211 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:21,211 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,211 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN 2024-12-05T13:17:21,211 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-05T13:17:21,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:21,212 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:21,212 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:21,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,220 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,220 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,220 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,220 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN to state=OFFLINE, location=null, table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3 to restore RIT 2024-12-05T13:17:21,221 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN to state=OFFLINE, location=null, table=test, region=cb80ccb4b3541210c6fda8b4b988adb5 to restore RIT 2024-12-05T13:17:21,221 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN to state=OFFLINE, location=null, table=test, region=3c03197a9f3300f90c023ccedb4f2112 to restore RIT 2024-12-05T13:17:21,221 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, ASSIGN to state=OFFLINE, location=null, table=test, region=eb2ca1f66d6c9b4513c79bf39e4e5bfe to restore RIT 2024-12-05T13:17:21,221 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404671221 2024-12-05T13:17:21,221 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:21,221 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,223 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:21,224 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:21,224 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:21,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,224 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,227 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,228 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,229 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,229 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:21,229 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404701229 2024-12-05T13:17:21,229 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404761229 2024-12-05T13:17:21,229 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-05T13:17:21,229 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:21,229 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,229 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:21,229 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:21,229 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:21,229 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:21,229 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:21,230 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:21,231 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T13:17:21,232 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:21,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:21,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,233 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:21,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:21,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:21,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:21,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:21,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:21,235 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:21,235 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,236 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,236 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,236 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,236 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,236 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,236 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,236 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,236 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:21,236 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,236 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:21,236 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,236 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN 2024-12-05T13:17:21,236 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-05T13:17:21,237 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:21,237 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:21,237 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:21,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,245 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,245 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,245 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,245 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN to state=OFFLINE, location=null, table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3 to restore RIT 2024-12-05T13:17:21,246 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN to state=OFFLINE, location=null, table=test, region=cb80ccb4b3541210c6fda8b4b988adb5 to restore RIT 2024-12-05T13:17:21,246 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3c03197a9f3300f90c023ccedb4f2112, ASSIGN to state=OFFLINE, location=null, table=test, region=3c03197a9f3300f90c023ccedb4f2112 to restore RIT 2024-12-05T13:17:21,246 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404671246 2024-12-05T13:17:21,246 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:21,246 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,248 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:21,249 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:21,249 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:21,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,249 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,253 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:21,253 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404701253 2024-12-05T13:17:21,253 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404761253 2024-12-05T13:17:21,253 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-05T13:17:21,253 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:21,253 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,253 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:21,253 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:21,254 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:21,254 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:21,254 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:21,254 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:21,255 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T13:17:21,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:21,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:21,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:21,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:21,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:21,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:21,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:21,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:21,258 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,259 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,259 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,259 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:21,259 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,259 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN 2024-12-05T13:17:21,259 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-05T13:17:21,259 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:21,260 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:21,260 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:21,279 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,294 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,324 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,324 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,324 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,325 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN to state=OFFLINE, location=null, table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3 to restore RIT 2024-12-05T13:17:21,325 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=cb80ccb4b3541210c6fda8b4b988adb5, ASSIGN to state=OFFLINE, location=null, table=test, region=cb80ccb4b3541210c6fda8b4b988adb5 to restore RIT 2024-12-05T13:17:21,326 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404671326 2024-12-05T13:17:21,326 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:21,326 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,327 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:21,331 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,331 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,331 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:21,332 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:21,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,332 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,332 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,332 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,332 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,334 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,335 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:21,335 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404701335 2024-12-05T13:17:21,335 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404761335 2024-12-05T13:17:21,335 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-05T13:17:21,335 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:21,335 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,336 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:21,336 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:21,336 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:21,336 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:21,336 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:21,336 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:21,337 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T13:17:21,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:21,339 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:21,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,340 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:21,343 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:21,344 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:21,344 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,344 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,344 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:21,344 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,344 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,344 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,344 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:21,344 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,344 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,344 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,344 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,344 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,344 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,344 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,344 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,345 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN 2024-12-05T13:17:21,345 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-05T13:17:21,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:21,346 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:21,346 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:21,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,362 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,362 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,362 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,364 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3, ASSIGN to state=OFFLINE, location=null, table=test, region=97426aa8ec496bfcaaf589a5cbe6f1b3 to restore RIT 2024-12-05T13:17:21,364 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404671364 2024-12-05T13:17:21,364 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:21,365 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,366 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:21,370 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:21,370 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,371 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,371 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,371 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,372 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,373 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,374 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,374 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,374 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:21,374 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404701374 2024-12-05T13:17:21,374 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404761374 2024-12-05T13:17:21,374 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-05T13:17:21,374 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:21,374 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,374 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:21,374 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:21,379 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:21,379 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:21,379 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:21,379 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:21,381 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-05T13:17:21,382 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:21,383 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,383 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,383 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,383 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:21,383 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,383 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,383 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:21,384 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:21,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:21,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:21,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,385 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,386 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,386 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,386 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,386 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,386 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,386 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,386 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,386 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-05T13:17:21,394 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:21,394 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:21,394 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:21,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,403 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,403 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,408 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,415 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404671415 2024-12-05T13:17:21,415 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:21,415 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,417 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:21,420 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:21,420 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,421 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,422 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,422 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,422 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,422 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,422 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,422 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,422 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,422 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,423 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,424 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,424 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:21,424 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404701424 2024-12-05T13:17:21,424 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404761424 2024-12-05T13:17:21,424 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-12-05T13:17:21,424 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:21,424 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,424 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:21,425 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:21,425 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:21,425 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:21,425 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:21,426 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:21,427 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,427 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-05T13:17:21,429 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:21,429 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,430 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,430 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,430 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:21,430 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,430 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:21,431 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:21,431 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,432 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:21,432 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,432 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,433 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:21,433 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,433 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,433 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,433 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,433 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,433 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,433 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,433 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.1733404634576, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,433 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-05T13:17:21,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:21,435 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:21,436 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:21,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,445 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,445 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,446 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,451 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404671451 2024-12-05T13:17:21,451 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:21,451 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,453 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:21,455 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:21,455 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,455 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,459 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,460 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,461 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,461 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:21,461 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404701461 2024-12-05T13:17:21,461 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404761461 2024-12-05T13:17:21,461 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-12-05T13:17:21,461 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:21,461 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,461 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:21,461 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:21,462 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:21,462 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:21,462 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:21,462 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:21,463 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T13:17:21,464 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:21,464 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,464 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,464 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,464 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-12-05T13:17:21,464 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:21,465 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:21,465 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:21,465 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,465 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,465 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:21,465 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,465 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,465 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,466 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,466 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,466 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,466 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-05T13:17:21,467 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:21,467 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:21,467 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:21,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,479 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,479 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,479 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,480 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42815 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,481 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "88aaa70743e7" port: 36055 start_code: 1733404632807 } transition { transition_code: OPENED region_info { region_id: 1733404635844 table_name { namespace: "default" qualifier: "test" } start_key: "iii" end_key: "jjj" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 78 initiating_master_active_time: 1733404632952 }; retry (#1) after 2004ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-12-05T13:17:21,491 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404671491 2024-12-05T13:17:21,491 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:21,491 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,492 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:21,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,497 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:21,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,497 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:21,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,497 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,499 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,499 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:21,499 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404701499 2024-12-05T13:17:21,499 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404761499 2024-12-05T13:17:21,499 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-05T13:17:21,499 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:21,499 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,499 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:21,499 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:21,500 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:21,500 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:21,500 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:21,500 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:21,501 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-05T13:17:21,503 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:21,503 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,503 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,503 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-12-05T13:17:21,503 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:21,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:21,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:21,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T13:17:21,505 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,505 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,505 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,505 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,506 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-05T13:17:21,507 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:21,507 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:21,507 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:21,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,516 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,516 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,516 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,518 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404671518 2024-12-05T13:17:21,518 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:21,518 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,521 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:21,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,527 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:21,527 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404701527 2024-12-05T13:17:21,527 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404761527 2024-12-05T13:17:21,527 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-12-05T13:17:21,528 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:21,529 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,529 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,777 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:21,777 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,777 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:21,778 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:21,778 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:21,778 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:21,779 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:21,779 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:21,781 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-05T13:17:21,783 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:21,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,784 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-12-05T13:17:21,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:21,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:21,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:21,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,786 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,786 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-05T13:17:21,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:21,787 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:21,787 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:21,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,795 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,795 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,795 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,796 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404671796 2024-12-05T13:17:21,796 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:21,796 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,797 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:21,798 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:21,798 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,798 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,801 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,802 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,803 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,803 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:21,803 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404701803 2024-12-05T13:17:21,803 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404761803 2024-12-05T13:17:21,803 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-05T13:17:21,803 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:21,803 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,803 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:21,803 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:21,804 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:21,804 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:21,804 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:21,804 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:21,805 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T13:17:21,806 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:21,806 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-12-05T13:17:21,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:21,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:21,807 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:21,807 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,807 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,807 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 88aaa70743e7%2C41495%2C1733404632848.meta.1733404635228.meta, worker=88aaa70743e7,39293,1733404632706 didn't hold the lock before restarting, skip acquiring lock. 2024-12-05T13:17:21,807 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-05T13:17:21,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:21,808 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:21,808 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:21,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,816 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,816 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,816 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,817 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404671817 2024-12-05T13:17:21,817 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:21,817 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,819 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:21,820 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-12-05T13:17:21,820 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:21,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,820 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,822 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,823 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,823 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:21,823 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404701823 2024-12-05T13:17:21,823 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404761823 2024-12-05T13:17:21,823 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 5 msec 2024-12-05T13:17:21,823 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-12-05T13:17:21,823 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,823 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:21,823 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:21,823 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-12-05T13:17:21,823 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-12-05T13:17:21,823 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-12-05T13:17:21,823 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T13:17:21,824 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T13:17:21,825 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-12-05T13:17:21,825 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T13:17:21,825 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-12-05T13:17:21,826 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true 2024-12-05T13:17:21,826 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-12-05T13:17:21,826 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-12-05T13:17:21,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=61 2024-12-05T13:17:21,827 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=88aaa70743e7,39293,1733404632706, table=hbase:meta, region=1588230740 2024-12-05T13:17:21,827 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88aaa70743e7,39293,1733404632706, state=OPEN 2024-12-05T13:17:21,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T13:17:21,837 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,837 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,837 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T13:17:21,837 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733404671837 2024-12-05T13:17:21,837 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T13:17:21,837 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-05T13:17:21,839 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-12-05T13:17:21,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=74, ppid=64, state=RUNNABLE, hasLock=true; OpenRegionProcedure 97426aa8ec496bfcaaf589a5cbe6f1b3, server=88aaa70743e7,39293,1733404632706:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 88aaa70743e7,41495,1733404632848, splitWal=true, meta=true exec-time=3.5260 sec 2024-12-05T13:17:21,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c9639ed41e308974b4199260665db3c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b3cc71809f62cd16a817f5a172f68ff, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=943cec28fed0bc3f97c304818b208e3f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f8705d72ae6f07bbf8fa8bfbf5a81d07, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8e77e301280f5f9879631fbf6a099514, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1218265d16f004d3f7900dc1adf62958, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4debf2a1857c920c4d806684d7cb0807, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7bf9cb128da6b1afbc569bd4b25eba2d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=01ef88de67b81e433e4128dc0a819605, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8d9d0bbe2316236e0d9afb8d72cd4099, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f29f7ad7d2427bd1681491cb1725214f, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3ea448375edac1598b6cdff464d477bf, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb2ca1f66d6c9b4513c79bf39e4e5bfe, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c086578586a3011e8fde7b2976cca84b, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd586ec5ee08f004669e3c48942b3638, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3c03197a9f3300f90c023ccedb4f2112, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5d414b07d03c5766c12220a94662f4ff, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a362f9452e2fdf5e9192ce69e889c30d, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=97426aa8ec496bfcaaf589a5cbe6f1b3, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb504ea56b81b22b5ca81b650f200ded, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f4b86b8085620be60c3a91043e8f4d72, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb80ccb4b3541210c6fda8b4b988adb5, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abced629a2bc26273048aa2f4b6f8d9c, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e805a0311eea683702ecc957a61057e2, regionState=OPEN, lastHost=88aaa70743e7,36055,1733404632807, regionLocation=88aaa70743e7,36055,1733404632807, openSeqNum=2 2024-12-05T13:17:21,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cabc686ba5a0c762c06d5a4d0024dd7b, regionState=OPENING, lastHost=88aaa70743e7,41495,1733404632848, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,845 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6f220ce002468367c756923f6fbc1fc1, regionState=OPEN, lastHost=88aaa70743e7,39293,1733404632706, regionLocation=88aaa70743e7,39293,1733404632706, openSeqNum=2 2024-12-05T13:17:21,845 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T13:17:21,845 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733404701845 2024-12-05T13:17:21,845 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733404761845 2024-12-05T13:17:21,845 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-12-05T13:17:21,860 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=347 (was 381), OpenFileDescriptor=620 (was 643), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=367 (was 382), ProcessCount=11 (was 11), AvailableMemoryMB=4382 (was 4546) 2024-12-05T13:17:21,860 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-05T13:17:21,860 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T13:17:21,860 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T13:17:21,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T13:17:21,861 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T13:17:21,861 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T13:17:21,861 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T13:17:21,861 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=780736434, stopped=false 2024-12-05T13:17:21,861 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=88aaa70743e7,42815,1733404631821 2024-12-05T13:17:21,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T13:17:21,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T13:17:21,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T13:17:21,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:21,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:21,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:21,870 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:21,870 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T13:17:21,870 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T13:17:21,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T13:17:21,871 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '88aaa70743e7,39293,1733404632706' ***** 2024-12-05T13:17:21,871 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T13:17:21,871 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '88aaa70743e7,36055,1733404632807' ***** 2024-12-05T13:17:21,871 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T13:17:21,871 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T13:17:21,871 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T13:17:21,871 INFO [RS:1;88aaa70743e7:36055 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T13:17:21,871 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T13:17:21,871 INFO [RS:1;88aaa70743e7:36055 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T13:17:21,871 INFO [RS:0;88aaa70743e7:39293 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T13:17:21,871 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T13:17:21,871 INFO [RS:0;88aaa70743e7:39293 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T13:17:21,872 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(3091): Received CLOSE for bd586ec5ee08f004669e3c48942b3638 2024-12-05T13:17:21,872 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T13:17:21,872 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(3091): Received CLOSE for f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:21,872 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T13:17:21,873 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(3091): Received CLOSE for 8e77e301280f5f9879631fbf6a099514 2024-12-05T13:17:21,873 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(3091): Received CLOSE for a362f9452e2fdf5e9192ce69e889c30d 2024-12-05T13:17:21,873 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T13:17:21,873 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(3091): Received CLOSE for 7bf9cb128da6b1afbc569bd4b25eba2d 2024-12-05T13:17:21,873 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(3091): Received CLOSE for e805a0311eea683702ecc957a61057e2 2024-12-05T13:17:21,873 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(3091): Received CLOSE for f29f7ad7d2427bd1681491cb1725214f 2024-12-05T13:17:21,873 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(3091): Received CLOSE for 943cec28fed0bc3f97c304818b208e3f 2024-12-05T13:17:21,873 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(3091): Received CLOSE for cb80ccb4b3541210c6fda8b4b988adb5 2024-12-05T13:17:21,873 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(3091): Received CLOSE for 3ea448375edac1598b6cdff464d477bf 2024-12-05T13:17:21,873 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(3091): Received CLOSE for f4b86b8085620be60c3a91043e8f4d72 2024-12-05T13:17:21,873 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(959): stopping server 88aaa70743e7,36055,1733404632807 2024-12-05T13:17:21,873 INFO [RS:1;88aaa70743e7:36055 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T13:17:21,873 INFO [RS:1;88aaa70743e7:36055 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;88aaa70743e7:36055. 2024-12-05T13:17:21,873 DEBUG [RS:1;88aaa70743e7:36055 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T13:17:21,873 DEBUG [RS:1;88aaa70743e7:36055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T13:17:21,874 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(1321): Waiting on 10 regions to close 2024-12-05T13:17:21,874 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bd586ec5ee08f004669e3c48942b3638, disabling compactions & flushes 2024-12-05T13:17:21,874 DEBUG [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(1351): Waiting on 01ef88de67b81e433e4128dc0a819605, 3ea448375edac1598b6cdff464d477bf, 7bf9cb128da6b1afbc569bd4b25eba2d, 8e77e301280f5f9879631fbf6a099514, 943cec28fed0bc3f97c304818b208e3f, a362f9452e2fdf5e9192ce69e889c30d, bd586ec5ee08f004669e3c48942b3638, cb80ccb4b3541210c6fda8b4b988adb5, e805a0311eea683702ecc957a61057e2, f29f7ad7d2427bd1681491cb1725214f, f4b86b8085620be60c3a91043e8f4d72 2024-12-05T13:17:21,874 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(3091): Received CLOSE for c086578586a3011e8fde7b2976cca84b 2024-12-05T13:17:21,874 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(3091): Received CLOSE for 97426aa8ec496bfcaaf589a5cbe6f1b3 2024-12-05T13:17:21,874 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1733404635844.bd586ec5ee08f004669e3c48942b3638. 2024-12-05T13:17:21,874 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(3091): Received CLOSE for eb2ca1f66d6c9b4513c79bf39e4e5bfe 2024-12-05T13:17:21,874 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(3091): Received CLOSE for 5d414b07d03c5766c12220a94662f4ff 2024-12-05T13:17:21,874 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1733404635844.bd586ec5ee08f004669e3c48942b3638. 2024-12-05T13:17:21,874 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(3091): Received CLOSE for abced629a2bc26273048aa2f4b6f8d9c 2024-12-05T13:17:21,874 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(3091): Received CLOSE for 8d9d0bbe2316236e0d9afb8d72cd4099 2024-12-05T13:17:21,875 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(3091): Received CLOSE for bb504ea56b81b22b5ca81b650f200ded 2024-12-05T13:17:21,875 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1733404635844.bd586ec5ee08f004669e3c48942b3638. after waiting 0 ms 2024-12-05T13:17:21,875 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(3091): Received CLOSE for 5c9639ed41e308974b4199260665db3c 2024-12-05T13:17:21,875 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(3091): Received CLOSE for 4debf2a1857c920c4d806684d7cb0807 2024-12-05T13:17:21,875 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1733404635844.bd586ec5ee08f004669e3c48942b3638. 2024-12-05T13:17:21,875 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(3091): Received CLOSE for 6f220ce002468367c756923f6fbc1fc1 2024-12-05T13:17:21,875 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(3091): Received CLOSE for 1218265d16f004d3f7900dc1adf62958 2024-12-05T13:17:21,875 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(3091): Received CLOSE for cabc686ba5a0c762c06d5a4d0024dd7b 2024-12-05T13:17:21,875 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(959): stopping server 88aaa70743e7,39293,1733404632706 2024-12-05T13:17:21,875 INFO [RS:0;88aaa70743e7:39293 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T13:17:21,875 INFO [RS:0;88aaa70743e7:39293 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;88aaa70743e7:39293. 2024-12-05T13:17:21,875 DEBUG [RS:0;88aaa70743e7:39293 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T13:17:21,875 DEBUG [RS:0;88aaa70743e7:39293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T13:17:21,875 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T13:17:21,875 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T13:17:21,875 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T13:17:21,875 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-05T13:17:21,876 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f8705d72ae6f07bbf8fa8bfbf5a81d07, disabling compactions & flushes 2024-12-05T13:17:21,876 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07. 2024-12-05T13:17:21,876 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07. 2024-12-05T13:17:21,876 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07. after waiting 0 ms 2024-12-05T13:17:21,876 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07. 2024-12-05T13:17:21,882 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-12-05T13:17:21,882 DEBUG [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(1351): Waiting on 1218265d16f004d3f7900dc1adf62958, 1588230740, 4debf2a1857c920c4d806684d7cb0807, 5c9639ed41e308974b4199260665db3c, 5d414b07d03c5766c12220a94662f4ff, 6f220ce002468367c756923f6fbc1fc1, 8d9d0bbe2316236e0d9afb8d72cd4099, 97426aa8ec496bfcaaf589a5cbe6f1b3, abced629a2bc26273048aa2f4b6f8d9c, bb504ea56b81b22b5ca81b650f200ded, c086578586a3011e8fde7b2976cca84b, cabc686ba5a0c762c06d5a4d0024dd7b, eb2ca1f66d6c9b4513c79bf39e4e5bfe, f8705d72ae6f07bbf8fa8bfbf5a81d07 2024-12-05T13:17:21,882 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T13:17:21,882 INFO [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T13:17:21,882 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T13:17:21,882 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T13:17:21,882 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T13:17:21,882 INFO [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-12-05T13:17:21,886 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/bd586ec5ee08f004669e3c48942b3638/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:21,887 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/f8705d72ae6f07bbf8fa8bfbf5a81d07/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:21,893 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1733404635844.bd586ec5ee08f004669e3c48942b3638. 2024-12-05T13:17:21,893 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bd586ec5ee08f004669e3c48942b3638: Waiting for close lock at 1733404641873Running coprocessor pre-close hooks at 1733404641874 (+1 ms)Disabling compacts and flushes for region at 1733404641874Disabling writes for close at 1733404641875 (+1 ms)Writing region close event to WAL at 1733404641882 (+7 ms)Running coprocessor post-close hooks at 1733404641888 (+6 ms)Closed at 1733404641893 (+5 ms) 2024-12-05T13:17:21,893 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1733404635844.bd586ec5ee08f004669e3c48942b3638. 2024-12-05T13:17:21,894 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8e77e301280f5f9879631fbf6a099514, disabling compactions & flushes 2024-12-05T13:17:21,894 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1733404635844.8e77e301280f5f9879631fbf6a099514. 2024-12-05T13:17:21,894 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1733404635844.8e77e301280f5f9879631fbf6a099514. 2024-12-05T13:17:21,894 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1733404635844.8e77e301280f5f9879631fbf6a099514. after waiting 0 ms 2024-12-05T13:17:21,894 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1733404635844.8e77e301280f5f9879631fbf6a099514. 2024-12-05T13:17:21,902 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/8e77e301280f5f9879631fbf6a099514/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:21,902 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07. 2024-12-05T13:17:21,902 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f8705d72ae6f07bbf8fa8bfbf5a81d07: Waiting for close lock at 1733404641876Running coprocessor pre-close hooks at 1733404641876Disabling compacts and flushes for region at 1733404641876Disabling writes for close at 1733404641876Writing region close event to WAL at 1733404641882 (+6 ms)Running coprocessor post-close hooks at 1733404641902 (+20 ms)Closed at 1733404641902 2024-12-05T13:17:21,902 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1733404635844.f8705d72ae6f07bbf8fa8bfbf5a81d07. 2024-12-05T13:17:21,903 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c086578586a3011e8fde7b2976cca84b, disabling compactions & flushes 2024-12-05T13:17:21,903 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1733404635844.c086578586a3011e8fde7b2976cca84b. 2024-12-05T13:17:21,903 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1733404635844.c086578586a3011e8fde7b2976cca84b. 2024-12-05T13:17:21,903 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1733404635844.c086578586a3011e8fde7b2976cca84b. after waiting 0 ms 2024-12-05T13:17:21,903 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1733404635844.c086578586a3011e8fde7b2976cca84b. 2024-12-05T13:17:21,903 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1733404635844.8e77e301280f5f9879631fbf6a099514. 2024-12-05T13:17:21,903 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8e77e301280f5f9879631fbf6a099514: Waiting for close lock at 1733404641894Running coprocessor pre-close hooks at 1733404641894Disabling compacts and flushes for region at 1733404641894Disabling writes for close at 1733404641894Writing region close event to WAL at 1733404641894Running coprocessor post-close hooks at 1733404641903 (+9 ms)Closed at 1733404641903 2024-12-05T13:17:21,903 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1733404635844.8e77e301280f5f9879631fbf6a099514. 2024-12-05T13:17:21,904 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a362f9452e2fdf5e9192ce69e889c30d, disabling compactions & flushes 2024-12-05T13:17:21,904 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1733404635844.a362f9452e2fdf5e9192ce69e889c30d. 2024-12-05T13:17:21,904 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1733404635844.a362f9452e2fdf5e9192ce69e889c30d. 2024-12-05T13:17:21,904 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1733404635844.a362f9452e2fdf5e9192ce69e889c30d. after waiting 0 ms 2024-12-05T13:17:21,904 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1733404635844.a362f9452e2fdf5e9192ce69e889c30d. 2024-12-05T13:17:21,914 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/.tmp/info/1218724de28d49fe8ac38f7ea4d197b9 is 138, key is test,bbb,1733404635844.8b3cc71809f62cd16a817f5a172f68ff./info:regioninfo/1733404640159/Put/seqid=0 2024-12-05T13:17:21,917 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/a362f9452e2fdf5e9192ce69e889c30d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:21,917 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1733404635844.a362f9452e2fdf5e9192ce69e889c30d. 2024-12-05T13:17:21,917 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a362f9452e2fdf5e9192ce69e889c30d: Waiting for close lock at 1733404641904Running coprocessor pre-close hooks at 1733404641904Disabling compacts and flushes for region at 1733404641904Disabling writes for close at 1733404641904Writing region close event to WAL at 1733404641911 (+7 ms)Running coprocessor post-close hooks at 1733404641917 (+6 ms)Closed at 1733404641917 2024-12-05T13:17:21,918 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1733404635844.a362f9452e2fdf5e9192ce69e889c30d. 2024-12-05T13:17:21,918 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7bf9cb128da6b1afbc569bd4b25eba2d, disabling compactions & flushes 2024-12-05T13:17:21,918 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1733404635844.7bf9cb128da6b1afbc569bd4b25eba2d. 2024-12-05T13:17:21,918 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1733404635844.7bf9cb128da6b1afbc569bd4b25eba2d. 2024-12-05T13:17:21,918 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1733404635844.7bf9cb128da6b1afbc569bd4b25eba2d. after waiting 0 ms 2024-12-05T13:17:21,918 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1733404635844.7bf9cb128da6b1afbc569bd4b25eba2d. 2024-12-05T13:17:21,919 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/c086578586a3011e8fde7b2976cca84b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:21,920 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1733404635844.c086578586a3011e8fde7b2976cca84b. 2024-12-05T13:17:21,920 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c086578586a3011e8fde7b2976cca84b: Waiting for close lock at 1733404641903Running coprocessor pre-close hooks at 1733404641903Disabling compacts and flushes for region at 1733404641903Disabling writes for close at 1733404641903Writing region close event to WAL at 1733404641911 (+8 ms)Running coprocessor post-close hooks at 1733404641920 (+9 ms)Closed at 1733404641920 2024-12-05T13:17:21,920 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1733404635844.c086578586a3011e8fde7b2976cca84b. 2024-12-05T13:17:21,920 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 97426aa8ec496bfcaaf589a5cbe6f1b3, disabling compactions & flushes 2024-12-05T13:17:21,920 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3. 2024-12-05T13:17:21,921 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3. 2024-12-05T13:17:21,921 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3. after waiting 0 ms 2024-12-05T13:17:21,921 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3. 2024-12-05T13:17:21,927 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/7bf9cb128da6b1afbc569bd4b25eba2d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:21,928 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1733404635844.7bf9cb128da6b1afbc569bd4b25eba2d. 2024-12-05T13:17:21,928 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7bf9cb128da6b1afbc569bd4b25eba2d: Waiting for close lock at 1733404641918Running coprocessor pre-close hooks at 1733404641918Disabling compacts and flushes for region at 1733404641918Disabling writes for close at 1733404641918Writing region close event to WAL at 1733404641923 (+5 ms)Running coprocessor post-close hooks at 1733404641928 (+5 ms)Closed at 1733404641928 2024-12-05T13:17:21,928 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1733404635844.7bf9cb128da6b1afbc569bd4b25eba2d. 2024-12-05T13:17:21,929 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e805a0311eea683702ecc957a61057e2, disabling compactions & flushes 2024-12-05T13:17:21,929 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1733404635844.e805a0311eea683702ecc957a61057e2. 2024-12-05T13:17:21,929 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1733404635844.e805a0311eea683702ecc957a61057e2. 2024-12-05T13:17:21,929 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1733404635844.e805a0311eea683702ecc957a61057e2. after waiting 0 ms 2024-12-05T13:17:21,929 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1733404635844.e805a0311eea683702ecc957a61057e2. 2024-12-05T13:17:21,935 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/97426aa8ec496bfcaaf589a5cbe6f1b3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:21,936 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3. 2024-12-05T13:17:21,936 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 97426aa8ec496bfcaaf589a5cbe6f1b3: Waiting for close lock at 1733404641920Running coprocessor pre-close hooks at 1733404641920Disabling compacts and flushes for region at 1733404641920Disabling writes for close at 1733404641921 (+1 ms)Writing region close event to WAL at 1733404641926 (+5 ms)Running coprocessor post-close hooks at 1733404641936 (+10 ms)Closed at 1733404641936 2024-12-05T13:17:21,936 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1733404635844.97426aa8ec496bfcaaf589a5cbe6f1b3. 2024-12-05T13:17:21,937 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eb2ca1f66d6c9b4513c79bf39e4e5bfe, disabling compactions & flushes 2024-12-05T13:17:21,937 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe. 2024-12-05T13:17:21,937 INFO [regionserver/88aaa70743e7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T13:17:21,937 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe. 2024-12-05T13:17:21,937 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe. after waiting 0 ms 2024-12-05T13:17:21,937 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe. 2024-12-05T13:17:21,939 INFO [regionserver/88aaa70743e7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T13:17:21,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741869_1045 (size=8195) 2024-12-05T13:17:21,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741869_1045 (size=8195) 2024-12-05T13:17:21,941 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/e805a0311eea683702ecc957a61057e2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:21,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741869_1045 (size=8195) 2024-12-05T13:17:21,942 INFO [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/.tmp/info/1218724de28d49fe8ac38f7ea4d197b9 2024-12-05T13:17:21,945 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1733404635844.e805a0311eea683702ecc957a61057e2. 2024-12-05T13:17:21,945 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e805a0311eea683702ecc957a61057e2: Waiting for close lock at 1733404641929Running coprocessor pre-close hooks at 1733404641929Disabling compacts and flushes for region at 1733404641929Disabling writes for close at 1733404641929Writing region close event to WAL at 1733404641934 (+5 ms)Running coprocessor post-close hooks at 1733404641945 (+11 ms)Closed at 1733404641945 2024-12-05T13:17:21,945 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1733404635844.e805a0311eea683702ecc957a61057e2. 2024-12-05T13:17:21,945 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f29f7ad7d2427bd1681491cb1725214f, disabling compactions & flushes 2024-12-05T13:17:21,945 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1733404635844.f29f7ad7d2427bd1681491cb1725214f. 2024-12-05T13:17:21,945 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/eb2ca1f66d6c9b4513c79bf39e4e5bfe/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:21,945 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1733404635844.f29f7ad7d2427bd1681491cb1725214f. 2024-12-05T13:17:21,946 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1733404635844.f29f7ad7d2427bd1681491cb1725214f. after waiting 0 ms 2024-12-05T13:17:21,946 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1733404635844.f29f7ad7d2427bd1681491cb1725214f. 2024-12-05T13:17:21,946 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe. 2024-12-05T13:17:21,946 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eb2ca1f66d6c9b4513c79bf39e4e5bfe: Waiting for close lock at 1733404641937Running coprocessor pre-close hooks at 1733404641937Disabling compacts and flushes for region at 1733404641937Disabling writes for close at 1733404641937Writing region close event to WAL at 1733404641941 (+4 ms)Running coprocessor post-close hooks at 1733404641946 (+5 ms)Closed at 1733404641946 2024-12-05T13:17:21,946 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1733404635844.eb2ca1f66d6c9b4513c79bf39e4e5bfe. 2024-12-05T13:17:21,947 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5d414b07d03c5766c12220a94662f4ff, disabling compactions & flushes 2024-12-05T13:17:21,947 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1733404635844.5d414b07d03c5766c12220a94662f4ff. 2024-12-05T13:17:21,947 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1733404635844.5d414b07d03c5766c12220a94662f4ff. 2024-12-05T13:17:21,947 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1733404635844.5d414b07d03c5766c12220a94662f4ff. after waiting 0 ms 2024-12-05T13:17:21,947 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1733404635844.5d414b07d03c5766c12220a94662f4ff. 2024-12-05T13:17:21,951 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/f29f7ad7d2427bd1681491cb1725214f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:21,951 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1733404635844.f29f7ad7d2427bd1681491cb1725214f. 2024-12-05T13:17:21,951 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f29f7ad7d2427bd1681491cb1725214f: Waiting for close lock at 1733404641945Running coprocessor pre-close hooks at 1733404641945Disabling compacts and flushes for region at 1733404641945Disabling writes for close at 1733404641946 (+1 ms)Writing region close event to WAL at 1733404641946Running coprocessor post-close hooks at 1733404641951 (+5 ms)Closed at 1733404641951 2024-12-05T13:17:21,952 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1733404635844.f29f7ad7d2427bd1681491cb1725214f. 2024-12-05T13:17:21,952 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 943cec28fed0bc3f97c304818b208e3f, disabling compactions & flushes 2024-12-05T13:17:21,952 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1733404635844.943cec28fed0bc3f97c304818b208e3f. 2024-12-05T13:17:21,952 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1733404635844.943cec28fed0bc3f97c304818b208e3f. 2024-12-05T13:17:21,952 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1733404635844.943cec28fed0bc3f97c304818b208e3f. after waiting 0 ms 2024-12-05T13:17:21,952 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1733404635844.943cec28fed0bc3f97c304818b208e3f. 2024-12-05T13:17:21,952 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/.tmp/info/1218724de28d49fe8ac38f7ea4d197b9 as hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/info/1218724de28d49fe8ac38f7ea4d197b9 2024-12-05T13:17:21,956 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/943cec28fed0bc3f97c304818b208e3f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:21,957 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1733404635844.943cec28fed0bc3f97c304818b208e3f. 2024-12-05T13:17:21,957 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 943cec28fed0bc3f97c304818b208e3f: Waiting for close lock at 1733404641952Running coprocessor pre-close hooks at 1733404641952Disabling compacts and flushes for region at 1733404641952Disabling writes for close at 1733404641952Writing region close event to WAL at 1733404641952Running coprocessor post-close hooks at 1733404641957 (+5 ms)Closed at 1733404641957 2024-12-05T13:17:21,957 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1733404635844.943cec28fed0bc3f97c304818b208e3f. 2024-12-05T13:17:21,958 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cb80ccb4b3541210c6fda8b4b988adb5, disabling compactions & flushes 2024-12-05T13:17:21,958 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5. 2024-12-05T13:17:21,958 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5. 2024-12-05T13:17:21,958 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5. after waiting 0 ms 2024-12-05T13:17:21,958 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5. 2024-12-05T13:17:21,958 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/5d414b07d03c5766c12220a94662f4ff/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:21,960 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1733404635844.5d414b07d03c5766c12220a94662f4ff. 2024-12-05T13:17:21,960 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5d414b07d03c5766c12220a94662f4ff: Waiting for close lock at 1733404641946Running coprocessor pre-close hooks at 1733404641946Disabling compacts and flushes for region at 1733404641946Disabling writes for close at 1733404641947 (+1 ms)Writing region close event to WAL at 1733404641948 (+1 ms)Running coprocessor post-close hooks at 1733404641960 (+12 ms)Closed at 1733404641960 2024-12-05T13:17:21,960 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1733404635844.5d414b07d03c5766c12220a94662f4ff. 2024-12-05T13:17:21,961 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing abced629a2bc26273048aa2f4b6f8d9c, disabling compactions & flushes 2024-12-05T13:17:21,961 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1733404635844.abced629a2bc26273048aa2f4b6f8d9c. 2024-12-05T13:17:21,961 INFO [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/info/1218724de28d49fe8ac38f7ea4d197b9, entries=24, sequenceid=71, filesize=8.0 K 2024-12-05T13:17:21,961 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1733404635844.abced629a2bc26273048aa2f4b6f8d9c. 2024-12-05T13:17:21,961 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1733404635844.abced629a2bc26273048aa2f4b6f8d9c. after waiting 0 ms 2024-12-05T13:17:21,961 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1733404635844.abced629a2bc26273048aa2f4b6f8d9c. 2024-12-05T13:17:21,962 INFO [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=71, compaction requested=false 2024-12-05T13:17:21,965 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/cb80ccb4b3541210c6fda8b4b988adb5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:21,967 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/abced629a2bc26273048aa2f4b6f8d9c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:21,967 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5. 2024-12-05T13:17:21,967 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cb80ccb4b3541210c6fda8b4b988adb5: Waiting for close lock at 1733404641957Running coprocessor pre-close hooks at 1733404641957Disabling compacts and flushes for region at 1733404641957Disabling writes for close at 1733404641958 (+1 ms)Writing region close event to WAL at 1733404641958Running coprocessor post-close hooks at 1733404641966 (+8 ms)Closed at 1733404641966 2024-12-05T13:17:21,967 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1733404635844.cb80ccb4b3541210c6fda8b4b988adb5. 2024-12-05T13:17:21,967 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3ea448375edac1598b6cdff464d477bf, disabling compactions & flushes 2024-12-05T13:17:21,967 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1733404635844.3ea448375edac1598b6cdff464d477bf. 2024-12-05T13:17:21,967 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1733404635844.3ea448375edac1598b6cdff464d477bf. 2024-12-05T13:17:21,967 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1733404635844.abced629a2bc26273048aa2f4b6f8d9c. 2024-12-05T13:17:21,967 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1733404635844.3ea448375edac1598b6cdff464d477bf. after waiting 0 ms 2024-12-05T13:17:21,967 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for abced629a2bc26273048aa2f4b6f8d9c: Waiting for close lock at 1733404641961Running coprocessor pre-close hooks at 1733404641961Disabling compacts and flushes for region at 1733404641961Disabling writes for close at 1733404641961Writing region close event to WAL at 1733404641961Running coprocessor post-close hooks at 1733404641967 (+6 ms)Closed at 1733404641967 2024-12-05T13:17:21,967 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1733404635844.3ea448375edac1598b6cdff464d477bf. 2024-12-05T13:17:21,968 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1733404635844.abced629a2bc26273048aa2f4b6f8d9c. 2024-12-05T13:17:21,968 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8d9d0bbe2316236e0d9afb8d72cd4099, disabling compactions & flushes 2024-12-05T13:17:21,968 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1733404635844.8d9d0bbe2316236e0d9afb8d72cd4099. 2024-12-05T13:17:21,968 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1733404635844.8d9d0bbe2316236e0d9afb8d72cd4099. 2024-12-05T13:17:21,968 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1733404635844.8d9d0bbe2316236e0d9afb8d72cd4099. after waiting 0 ms 2024-12-05T13:17:21,968 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1733404635844.8d9d0bbe2316236e0d9afb8d72cd4099. 2024-12-05T13:17:21,970 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-12-05T13:17:21,971 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T13:17:21,971 INFO [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T13:17:21,971 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733404641882Running coprocessor pre-close hooks at 1733404641882Disabling compacts and flushes for region at 1733404641882Disabling writes for close at 1733404641882Obtaining lock to block concurrent updates at 1733404641882Preparing flush snapshotting stores in 1588230740 at 1733404641882Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1733404641893 (+11 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733404641895 (+2 ms)Flushing 1588230740/info: creating writer at 1733404641895Flushing 1588230740/info: appending metadata at 1733404641914 (+19 ms)Flushing 1588230740/info: closing flushed file at 1733404641914Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3aa189a9: reopening flushed file at 1733404641951 (+37 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=71, compaction requested=false at 1733404641962 (+11 ms)Writing region close event to WAL at 1733404641965 (+3 ms)Running coprocessor post-close hooks at 1733404641971 (+6 ms)Closed at 1733404641971 2024-12-05T13:17:21,971 DEBUG [RS_CLOSE_META-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T13:17:21,971 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/8d9d0bbe2316236e0d9afb8d72cd4099/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:21,973 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1733404635844.8d9d0bbe2316236e0d9afb8d72cd4099. 2024-12-05T13:17:21,973 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8d9d0bbe2316236e0d9afb8d72cd4099: Waiting for close lock at 1733404641968Running coprocessor pre-close hooks at 1733404641968Disabling compacts and flushes for region at 1733404641968Disabling writes for close at 1733404641968Writing region close event to WAL at 1733404641968Running coprocessor post-close hooks at 1733404641973 (+5 ms)Closed at 1733404641973 2024-12-05T13:17:21,973 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1733404635844.8d9d0bbe2316236e0d9afb8d72cd4099. 2024-12-05T13:17:21,973 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bb504ea56b81b22b5ca81b650f200ded, disabling compactions & flushes 2024-12-05T13:17:21,973 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1733404635844.bb504ea56b81b22b5ca81b650f200ded. 2024-12-05T13:17:21,973 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1733404635844.bb504ea56b81b22b5ca81b650f200ded. 2024-12-05T13:17:21,973 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1733404635844.bb504ea56b81b22b5ca81b650f200ded. after waiting 0 ms 2024-12-05T13:17:21,973 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1733404635844.bb504ea56b81b22b5ca81b650f200ded. 2024-12-05T13:17:21,987 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/3ea448375edac1598b6cdff464d477bf/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:21,988 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1733404635844.3ea448375edac1598b6cdff464d477bf. 2024-12-05T13:17:21,988 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3ea448375edac1598b6cdff464d477bf: Waiting for close lock at 1733404641967Running coprocessor pre-close hooks at 1733404641967Disabling compacts and flushes for region at 1733404641967Disabling writes for close at 1733404641967Writing region close event to WAL at 1733404641968 (+1 ms)Running coprocessor post-close hooks at 1733404641988 (+20 ms)Closed at 1733404641988 2024-12-05T13:17:21,989 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1733404635844.3ea448375edac1598b6cdff464d477bf. 2024-12-05T13:17:21,989 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f4b86b8085620be60c3a91043e8f4d72, disabling compactions & flushes 2024-12-05T13:17:21,989 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1733404635844.f4b86b8085620be60c3a91043e8f4d72. 2024-12-05T13:17:21,989 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1733404635844.f4b86b8085620be60c3a91043e8f4d72. 2024-12-05T13:17:21,989 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1733404635844.f4b86b8085620be60c3a91043e8f4d72. after waiting 0 ms 2024-12-05T13:17:21,989 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1733404635844.f4b86b8085620be60c3a91043e8f4d72. 2024-12-05T13:17:21,999 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/bb504ea56b81b22b5ca81b650f200ded/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:21,999 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/f4b86b8085620be60c3a91043e8f4d72/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:22,000 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1733404635844.f4b86b8085620be60c3a91043e8f4d72. 2024-12-05T13:17:22,000 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1733404635844.bb504ea56b81b22b5ca81b650f200ded. 2024-12-05T13:17:22,000 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f4b86b8085620be60c3a91043e8f4d72: Waiting for close lock at 1733404641989Running coprocessor pre-close hooks at 1733404641989Disabling compacts and flushes for region at 1733404641989Disabling writes for close at 1733404641989Writing region close event to WAL at 1733404641989Running coprocessor post-close hooks at 1733404642000 (+11 ms)Closed at 1733404642000 2024-12-05T13:17:22,000 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bb504ea56b81b22b5ca81b650f200ded: Waiting for close lock at 1733404641973Running coprocessor pre-close hooks at 1733404641973Disabling compacts and flushes for region at 1733404641973Disabling writes for close at 1733404641973Writing region close event to WAL at 1733404641974 (+1 ms)Running coprocessor post-close hooks at 1733404642000 (+26 ms)Closed at 1733404642000 2024-12-05T13:17:22,000 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1733404635844.f4b86b8085620be60c3a91043e8f4d72. 2024-12-05T13:17:22,000 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1733404635844.bb504ea56b81b22b5ca81b650f200ded. 2024-12-05T13:17:22,000 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5c9639ed41e308974b4199260665db3c, disabling compactions & flushes 2024-12-05T13:17:22,000 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1733404635844.5c9639ed41e308974b4199260665db3c. 2024-12-05T13:17:22,001 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1733404635844.5c9639ed41e308974b4199260665db3c. 2024-12-05T13:17:22,001 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1733404635844.5c9639ed41e308974b4199260665db3c. after waiting 0 ms 2024-12-05T13:17:22,001 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1733404635844.5c9639ed41e308974b4199260665db3c. 2024-12-05T13:17:22,005 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/5c9639ed41e308974b4199260665db3c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:22,006 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1733404635844.5c9639ed41e308974b4199260665db3c. 2024-12-05T13:17:22,006 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5c9639ed41e308974b4199260665db3c: Waiting for close lock at 1733404642000Running coprocessor pre-close hooks at 1733404642000Disabling compacts and flushes for region at 1733404642000Disabling writes for close at 1733404642001 (+1 ms)Writing region close event to WAL at 1733404642001Running coprocessor post-close hooks at 1733404642006 (+5 ms)Closed at 1733404642006 2024-12-05T13:17:22,006 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1733404635844.5c9639ed41e308974b4199260665db3c. 2024-12-05T13:17:22,006 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4debf2a1857c920c4d806684d7cb0807, disabling compactions & flushes 2024-12-05T13:17:22,006 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1733404635844.4debf2a1857c920c4d806684d7cb0807. 2024-12-05T13:17:22,007 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1733404635844.4debf2a1857c920c4d806684d7cb0807. 2024-12-05T13:17:22,007 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1733404635844.4debf2a1857c920c4d806684d7cb0807. after waiting 0 ms 2024-12-05T13:17:22,007 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1733404635844.4debf2a1857c920c4d806684d7cb0807. 2024-12-05T13:17:22,011 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/4debf2a1857c920c4d806684d7cb0807/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:22,012 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1733404635844.4debf2a1857c920c4d806684d7cb0807. 2024-12-05T13:17:22,012 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4debf2a1857c920c4d806684d7cb0807: Waiting for close lock at 1733404642006Running coprocessor pre-close hooks at 1733404642006Disabling compacts and flushes for region at 1733404642006Disabling writes for close at 1733404642007 (+1 ms)Writing region close event to WAL at 1733404642007Running coprocessor post-close hooks at 1733404642012 (+5 ms)Closed at 1733404642012 2024-12-05T13:17:22,012 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1733404635844.4debf2a1857c920c4d806684d7cb0807. 2024-12-05T13:17:22,013 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6f220ce002468367c756923f6fbc1fc1, disabling compactions & flushes 2024-12-05T13:17:22,013 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1733404635844.6f220ce002468367c756923f6fbc1fc1. 2024-12-05T13:17:22,013 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1733404635844.6f220ce002468367c756923f6fbc1fc1. 2024-12-05T13:17:22,013 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1733404635844.6f220ce002468367c756923f6fbc1fc1. after waiting 0 ms 2024-12-05T13:17:22,013 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1733404635844.6f220ce002468367c756923f6fbc1fc1. 2024-12-05T13:17:22,017 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/6f220ce002468367c756923f6fbc1fc1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:22,018 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1733404635844.6f220ce002468367c756923f6fbc1fc1. 2024-12-05T13:17:22,018 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6f220ce002468367c756923f6fbc1fc1: Waiting for close lock at 1733404642012Running coprocessor pre-close hooks at 1733404642012Disabling compacts and flushes for region at 1733404642012Disabling writes for close at 1733404642013 (+1 ms)Writing region close event to WAL at 1733404642013Running coprocessor post-close hooks at 1733404642018 (+5 ms)Closed at 1733404642018 2024-12-05T13:17:22,019 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1733404635844.6f220ce002468367c756923f6fbc1fc1. 2024-12-05T13:17:22,019 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1218265d16f004d3f7900dc1adf62958, disabling compactions & flushes 2024-12-05T13:17:22,019 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1733404635844.1218265d16f004d3f7900dc1adf62958. 2024-12-05T13:17:22,019 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1733404635844.1218265d16f004d3f7900dc1adf62958. 2024-12-05T13:17:22,019 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1733404635844.1218265d16f004d3f7900dc1adf62958. after waiting 0 ms 2024-12-05T13:17:22,019 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1733404635844.1218265d16f004d3f7900dc1adf62958. 2024-12-05T13:17:22,025 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/1218265d16f004d3f7900dc1adf62958/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:22,026 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1733404635844.1218265d16f004d3f7900dc1adf62958. 2024-12-05T13:17:22,026 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1218265d16f004d3f7900dc1adf62958: Waiting for close lock at 1733404642019Running coprocessor pre-close hooks at 1733404642019Disabling compacts and flushes for region at 1733404642019Disabling writes for close at 1733404642019Writing region close event to WAL at 1733404642020 (+1 ms)Running coprocessor post-close hooks at 1733404642026 (+6 ms)Closed at 1733404642026 2024-12-05T13:17:22,026 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1733404635844.1218265d16f004d3f7900dc1adf62958. 2024-12-05T13:17:22,026 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cabc686ba5a0c762c06d5a4d0024dd7b, disabling compactions & flushes 2024-12-05T13:17:22,026 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b. 2024-12-05T13:17:22,026 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b. 2024-12-05T13:17:22,026 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b. after waiting 0 ms 2024-12-05T13:17:22,026 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b. 2024-12-05T13:17:22,032 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/data/default/test/cabc686ba5a0c762c06d5a4d0024dd7b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T13:17:22,032 INFO [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b. 2024-12-05T13:17:22,033 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cabc686ba5a0c762c06d5a4d0024dd7b: Waiting for close lock at 1733404642026Running coprocessor pre-close hooks at 1733404642026Disabling compacts and flushes for region at 1733404642026Disabling writes for close at 1733404642026Writing region close event to WAL at 1733404642028 (+2 ms)Running coprocessor post-close hooks at 1733404642032 (+4 ms)Closed at 1733404642032 2024-12-05T13:17:22,033 DEBUG [RS_CLOSE_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1733404635844.cabc686ba5a0c762c06d5a4d0024dd7b. 2024-12-05T13:17:22,075 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(976): stopping server 88aaa70743e7,36055,1733404632807; all regions closed. 2024-12-05T13:17:22,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741833_1009 (size=5731) 2024-12-05T13:17:22,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741833_1009 (size=5731) 2024-12-05T13:17:22,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741833_1009 (size=5731) 2024-12-05T13:17:22,081 DEBUG [RS:1;88aaa70743e7:36055 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/oldWALs 2024-12-05T13:17:22,082 INFO [RS:1;88aaa70743e7:36055 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 88aaa70743e7%2C36055%2C1733404632807:(num 1733404634576) 2024-12-05T13:17:22,082 DEBUG [RS:1;88aaa70743e7:36055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T13:17:22,082 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T13:17:22,082 INFO [RS:1;88aaa70743e7:36055 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T13:17:22,082 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(976): stopping server 88aaa70743e7,39293,1733404632706; all regions closed. 2024-12-05T13:17:22,082 INFO [RS:1;88aaa70743e7:36055 {}] hbase.ChoreService(370): Chore service for: regionserver/88aaa70743e7:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-05T13:17:22,082 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T13:17:22,082 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T13:17:22,082 INFO [regionserver/88aaa70743e7:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T13:17:22,082 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T13:17:22,082 INFO [RS:1;88aaa70743e7:36055 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T13:17:22,082 WARN [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:17:22,083 WARN [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,iii,1733404635844.01ef88de67b81e433e4128dc0a819605., aborting... java.io.IOException: Failed to report opened region to master: test,iii,1733404635844.01ef88de67b81e433e4128dc0a819605. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:17:22,083 ERROR [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2440): ***** ABORTING region server 88aaa70743e7,36055,1733404632807: Failed to open region test,iii,1733404635844.01ef88de67b81e433e4128dc0a819605. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,iii,1733404635844.01ef88de67b81e433e4128dc0a819605. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T13:17:22,083 ERROR [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-05T13:17:22,083 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-05T13:17:22,084 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-05T13:17:22,084 INFO [RS:1;88aaa70743e7:36055 {}] executor.ExecutorService(108): Executor-10-RS_OPEN_REGION-regionserver/88aaa70743e7:0 had [Event #37 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #38 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler)] on shutdown 2024-12-05T13:17:22,085 INFO [RS:1;88aaa70743e7:36055 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36055 2024-12-05T13:17:22,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741865_1041 (size=4204) 2024-12-05T13:17:22,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741865_1041 (size=4204) 2024-12-05T13:17:22,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741865_1041 (size=4204) 2024-12-05T13:17:22,087 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-05T13:17:22,088 DEBUG [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-05T13:17:22,089 DEBUG [RS:0;88aaa70743e7:39293 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/oldWALs 2024-12-05T13:17:22,089 INFO [RS:0;88aaa70743e7:39293 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 88aaa70743e7%2C39293%2C1733404632706.meta:.meta(num 1733404639370) 2024-12-05T13:17:22,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741835_1011 (size=7064) 2024-12-05T13:17:22,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741835_1011 (size=7064) 2024-12-05T13:17:22,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741835_1011 (size=7064) 2024-12-05T13:17:22,094 DEBUG [RS:0;88aaa70743e7:39293 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/oldWALs 2024-12-05T13:17:22,094 INFO [RS:0;88aaa70743e7:39293 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 88aaa70743e7%2C39293%2C1733404632706:(num 1733404634577) 2024-12-05T13:17:22,095 DEBUG [RS:0;88aaa70743e7:39293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T13:17:22,095 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T13:17:22,095 INFO [RS:0;88aaa70743e7:39293 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T13:17:22,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/88aaa70743e7,36055,1733404632807 2024-12-05T13:17:22,095 INFO [RS:0;88aaa70743e7:39293 {}] hbase.ChoreService(370): Chore service for: regionserver/88aaa70743e7:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-05T13:17:22,095 INFO [RS:1;88aaa70743e7:36055 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T13:17:22,095 INFO [RS:0;88aaa70743e7:39293 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T13:17:22,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T13:17:22,095 INFO [RS:0;88aaa70743e7:39293 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39293 2024-12-05T13:17:22,095 INFO [regionserver/88aaa70743e7:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T13:17:22,097 INFO [RS_OPEN_REGION-regionserver/88aaa70743e7:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 476977776 }, "NonHeapMemoryUsage": { "committed": 163643392, "init": 7667712, "max": -1, "used": 161056736 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "88aaa70743e7", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 38528, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 115, "ProcessCallTime_min": 0, "ProcessCallTime_max": 39, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 9, "ProcessCallTime_median": 19, "ProcessCallTime_75th_percentile": 29, "ProcessCallTime_90th_percentile": 35, "ProcessCallTime_95th_percentile": 37, "ProcessCallTime_98th_percentile": 38, "ProcessCallTime_99th_percentile": 38, "ProcessCallTime_99.9th_percentile": 38, "ProcessCallTime_TimeRangeCount_0-1": 115, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 115, "QueueCallTime_min": 0, "QueueCallTime_max": 15, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 3, "QueueCallTime_median": 7, "QueueCallTime_75th_percentile": 11, "QueueCallTime_90th_percentile": 13, "QueueCallTime_95th_percentile": 14, "QueueCallTime_98th_percentile": 14, "QueueCallTime_99th_percentile": 14, "QueueCallTime_99.9th_percentile": 14, "QueueCallTime_TimeRangeCount_0-1": 115, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 115, "TotalCallTime_min": 0, "TotalCallTime_max": 39, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 9, "TotalCallTime_median": 19, "TotalCallTime_75th_percentile": 29, "TotalCallTime_90th_percentile": 35, "TotalCallTime_95th_percentile": 37, "TotalCallTime_98th_percentile": 38, "TotalCallTime_99th_percentile": 38, "TotalCallTime_99.9th_percentile": 38, "TotalCallTime_TimeRangeCount_0-1": 115, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 115, "ResponseSize_min": 0, "ResponseSize_max": 22855, "ResponseSize_mean": 1787, "ResponseSize_25th_percentile": 5713, "ResponseSize_median": 11427, "ResponseSize_75th_percentile": 17141, "ResponseSize_90th_percentile": 20569, "ResponseSize_95th_percentile": 21712, "ResponseSize_98th_percentile": 22397, "ResponseSize_99th_percentile": 22626, "ResponseSize_99.9th_percentile": 22832, "ResponseSize_SizeRangeCount_0-10": 115, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 115, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 303, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 115, "sentBytes": 160506 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "88aaa70743e7", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:50894", "tag.serverName": "88aaa70743e7,36055,1733404632807", "tag.clusterId": "d611f681-e8be-43c0-b7da-5bfaf5772f57", "tag.Context": "regionserver", "tag.Hostname": "88aaa70743e7", "regionCount": 10, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733404632807, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3332, "localBytesRead": 3332, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 36711, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 9, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 9, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 9, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-05T13:17:22,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/88aaa70743e7,39293,1733404632706 2024-12-05T13:17:22,103 INFO [RS:0;88aaa70743e7:39293 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T13:17:22,103 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [88aaa70743e7,39293,1733404632706] 2024-12-05T13:17:22,111 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/88aaa70743e7,39293,1733404632706 already deleted, retry=false 2024-12-05T13:17:22,111 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 88aaa70743e7,39293,1733404632706 expired; onlineServers=1 2024-12-05T13:17:22,111 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [88aaa70743e7,36055,1733404632807] 2024-12-05T13:17:22,120 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/88aaa70743e7,36055,1733404632807 already deleted, retry=false 2024-12-05T13:17:22,120 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 88aaa70743e7,36055,1733404632807 expired; onlineServers=0 2024-12-05T13:17:22,120 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '88aaa70743e7,42815,1733404631821' ***** 2024-12-05T13:17:22,120 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T13:17:22,120 INFO [M:0;88aaa70743e7:42815 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T13:17:22,120 INFO [M:0;88aaa70743e7:42815 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T13:17:22,120 DEBUG [M:0;88aaa70743e7:42815 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T13:17:22,120 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T13:17:22,120 DEBUG [M:0;88aaa70743e7:42815 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T13:17:22,120 DEBUG [master/88aaa70743e7:0:becomeActiveMaster-HFileCleaner.large.0-1733404634141 {}] cleaner.HFileCleaner(306): Exit Thread[master/88aaa70743e7:0:becomeActiveMaster-HFileCleaner.large.0-1733404634141,5,FailOnTimeoutGroup] 2024-12-05T13:17:22,120 DEBUG [master/88aaa70743e7:0:becomeActiveMaster-HFileCleaner.small.0-1733404634149 {}] cleaner.HFileCleaner(306): Exit Thread[master/88aaa70743e7:0:becomeActiveMaster-HFileCleaner.small.0-1733404634149,5,FailOnTimeoutGroup] 2024-12-05T13:17:22,121 INFO [M:0;88aaa70743e7:42815 {}] hbase.ChoreService(370): Chore service for: master/88aaa70743e7:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-05T13:17:22,121 INFO [M:0;88aaa70743e7:42815 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T13:17:22,121 DEBUG [M:0;88aaa70743e7:42815 {}] master.HMaster(1795): Stopping service threads 2024-12-05T13:17:22,121 INFO [M:0;88aaa70743e7:42815 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T13:17:22,121 INFO [M:0;88aaa70743e7:42815 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T13:17:22,121 INFO [M:0;88aaa70743e7:42815 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T13:17:22,122 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T13:17:22,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T13:17:22,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T13:17:22,128 DEBUG [M:0;88aaa70743e7:42815 {}] zookeeper.ZKUtil(347): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T13:17:22,128 WARN [M:0;88aaa70743e7:42815 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T13:17:22,129 INFO [M:0;88aaa70743e7:42815 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/.lastflushedseqids 2024-12-05T13:17:22,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741870_1046 (size=1628) 2024-12-05T13:17:22,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741870_1046 (size=1628) 2024-12-05T13:17:22,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741870_1046 (size=1628) 2024-12-05T13:17:22,144 INFO [M:0;88aaa70743e7:42815 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T13:17:22,144 INFO [M:0;88aaa70743e7:42815 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T13:17:22,144 DEBUG [M:0;88aaa70743e7:42815 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T13:17:22,144 INFO [M:0;88aaa70743e7:42815 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T13:17:22,144 DEBUG [M:0;88aaa70743e7:42815 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T13:17:22,144 DEBUG [M:0;88aaa70743e7:42815 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T13:17:22,144 DEBUG [M:0;88aaa70743e7:42815 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T13:17:22,144 INFO [M:0;88aaa70743e7:42815 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=249.11 KB heapSize=302.72 KB 2024-12-05T13:17:22,159 DEBUG [M:0;88aaa70743e7:42815 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7f1af94c9c4842d1ad71d2791bfb435c is 82, key is hbase:meta,,1/info:regioninfo/1733404639708/Put/seqid=0 2024-12-05T13:17:22,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741871_1047 (size=6071) 2024-12-05T13:17:22,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741871_1047 (size=6071) 2024-12-05T13:17:22,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741871_1047 (size=6071) 2024-12-05T13:17:22,166 INFO [M:0;88aaa70743e7:42815 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7f1af94c9c4842d1ad71d2791bfb435c 2024-12-05T13:17:22,195 DEBUG [M:0;88aaa70743e7:42815 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2ba590a93a044f01838f87343cba3f9b is 2990, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1733404641839/Put/seqid=0 2024-12-05T13:17:22,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741872_1048 (size=12801) 2024-12-05T13:17:22,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741872_1048 (size=12801) 2024-12-05T13:17:22,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741872_1048 (size=12801) 2024-12-05T13:17:22,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T13:17:22,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36055-0x101a6d1666f0002, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T13:17:22,206 INFO [RS:1;88aaa70743e7:36055 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T13:17:22,206 INFO [RS:1;88aaa70743e7:36055 {}] regionserver.HRegionServer(1031): Exiting; stopping=88aaa70743e7,36055,1733404632807; zookeeper connection closed. 2024-12-05T13:17:22,206 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3a5c2673 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3a5c2673 2024-12-05T13:17:22,206 INFO [M:0;88aaa70743e7:42815 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=247.83 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2ba590a93a044f01838f87343cba3f9b 2024-12-05T13:17:22,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T13:17:22,211 INFO [RS:0;88aaa70743e7:39293 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T13:17:22,211 INFO [RS:0;88aaa70743e7:39293 {}] regionserver.HRegionServer(1031): Exiting; stopping=88aaa70743e7,39293,1733404632706; zookeeper connection closed. 2024-12-05T13:17:22,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39293-0x101a6d1666f0001, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T13:17:22,212 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@a0e5141 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@a0e5141 2024-12-05T13:17:22,212 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-05T13:17:22,229 DEBUG [M:0;88aaa70743e7:42815 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5a53cb5ade054506864d337f96127f2e is 69, key is 88aaa70743e7,36055,1733404632807/rs:state/1733404634268/Put/seqid=0 2024-12-05T13:17:22,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741873_1049 (size=5442) 2024-12-05T13:17:22,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741873_1049 (size=5442) 2024-12-05T13:17:22,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741873_1049 (size=5442) 2024-12-05T13:17:22,237 INFO [M:0;88aaa70743e7:42815 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5a53cb5ade054506864d337f96127f2e 2024-12-05T13:17:22,244 INFO [M:0;88aaa70743e7:42815 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5a53cb5ade054506864d337f96127f2e 2024-12-05T13:17:22,258 DEBUG [M:0;88aaa70743e7:42815 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5554b6944c99497bb9647e11d3ee80b9 is 52, key is load_balancer_on/state:d/1733404638221/Put/seqid=0 2024-12-05T13:17:22,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741874_1050 (size=5058) 2024-12-05T13:17:22,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741874_1050 (size=5058) 2024-12-05T13:17:22,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741874_1050 (size=5058) 2024-12-05T13:17:22,266 INFO [M:0;88aaa70743e7:42815 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5554b6944c99497bb9647e11d3ee80b9 2024-12-05T13:17:22,273 DEBUG [M:0;88aaa70743e7:42815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7f1af94c9c4842d1ad71d2791bfb435c as hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7f1af94c9c4842d1ad71d2791bfb435c 2024-12-05T13:17:22,275 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-05T13:17:22,275 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-05T13:17:22,276 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test 2024-12-05T13:17:22,276 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test Metrics about Tables on a single HBase RegionServer 2024-12-05T13:17:22,277 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T13:17:22,277 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-05T13:17:22,278 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-05T13:17:22,278 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-05T13:17:22,281 INFO [M:0;88aaa70743e7:42815 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7f1af94c9c4842d1ad71d2791bfb435c, entries=14, sequenceid=572, filesize=5.9 K 2024-12-05T13:17:22,282 DEBUG [M:0;88aaa70743e7:42815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2ba590a93a044f01838f87343cba3f9b as hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2ba590a93a044f01838f87343cba3f9b 2024-12-05T13:17:22,291 INFO [M:0;88aaa70743e7:42815 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2ba590a93a044f01838f87343cba3f9b, entries=79, sequenceid=572, filesize=12.5 K 2024-12-05T13:17:22,292 DEBUG [M:0;88aaa70743e7:42815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5a53cb5ade054506864d337f96127f2e as hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5a53cb5ade054506864d337f96127f2e 2024-12-05T13:17:22,300 INFO [M:0;88aaa70743e7:42815 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5a53cb5ade054506864d337f96127f2e 2024-12-05T13:17:22,300 INFO [M:0;88aaa70743e7:42815 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5a53cb5ade054506864d337f96127f2e, entries=3, sequenceid=572, filesize=5.3 K 2024-12-05T13:17:22,302 DEBUG [M:0;88aaa70743e7:42815 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5554b6944c99497bb9647e11d3ee80b9 as hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5554b6944c99497bb9647e11d3ee80b9 2024-12-05T13:17:22,308 INFO [M:0;88aaa70743e7:42815 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43169/user/jenkins/test-data/03b6ba3c-73d1-7c39-f1a6-0f296e958f75/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5554b6944c99497bb9647e11d3ee80b9, entries=1, sequenceid=572, filesize=4.9 K 2024-12-05T13:17:22,309 INFO [M:0;88aaa70743e7:42815 {}] regionserver.HRegion(3140): Finished flush of dataSize ~249.11 KB/255088, heapSize ~302.66 KB/309920, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 165ms, sequenceid=572, compaction requested=false 2024-12-05T13:17:22,311 INFO [M:0;88aaa70743e7:42815 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T13:17:22,311 DEBUG [M:0;88aaa70743e7:42815 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733404642144Disabling compacts and flushes for region at 1733404642144Disabling writes for close at 1733404642144Obtaining lock to block concurrent updates at 1733404642144Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733404642144Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=255088, getHeapSize=309920, getOffHeapSize=0, getCellsCount=711 at 1733404642145 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733404642145Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733404642145Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733404642158 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733404642158Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733404642173 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733404642195 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733404642195Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733404642214 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733404642229 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733404642229Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733404642244 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733404642258 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733404642258Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2853841e: reopening flushed file at 1733404642272 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3eff6b1: reopening flushed file at 1733404642281 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ca4ea7f: reopening flushed file at 1733404642291 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e85da05: reopening flushed file at 1733404642300 (+9 ms)Finished flush of dataSize ~249.11 KB/255088, heapSize ~302.66 KB/309920, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 165ms, sequenceid=572, compaction requested=false at 1733404642309 (+9 ms)Writing region close event to WAL at 1733404642311 (+2 ms)Closed at 1733404642311 2024-12-05T13:17:22,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42229 is added to blk_1073741830_1006 (size=293277) 2024-12-05T13:17:22,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32831 is added to blk_1073741830_1006 (size=293277) 2024-12-05T13:17:22,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34109 is added to blk_1073741830_1006 (size=293277) 2024-12-05T13:17:22,314 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T13:17:22,314 INFO [M:0;88aaa70743e7:42815 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-05T13:17:22,314 INFO [M:0;88aaa70743e7:42815 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42815 2024-12-05T13:17:22,314 INFO [M:0;88aaa70743e7:42815 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T13:17:22,424 INFO [M:0;88aaa70743e7:42815 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T13:17:22,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T13:17:22,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42815-0x101a6d1666f0000, quorum=127.0.0.1:50894, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T13:17:22,428 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@46041a9d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T13:17:22,431 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2a39ea25{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T13:17:22,431 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T13:17:22,431 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@48a2d1c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T13:17:22,431 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6ed43cd1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/hadoop.log.dir/,STOPPED} 2024-12-05T13:17:22,434 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T13:17:22,434 WARN [BP-1141445696-172.17.0.3-1733404627457 heartbeating to localhost/127.0.0.1:43169 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T13:17:22,434 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T13:17:22,434 WARN [BP-1141445696-172.17.0.3-1733404627457 heartbeating to localhost/127.0.0.1:43169 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1141445696-172.17.0.3-1733404627457 (Datanode Uuid 8f66d2e2-73bf-4e91-9c27-85f4257afcf9) service to localhost/127.0.0.1:43169 2024-12-05T13:17:22,435 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/cluster_75ee48d7-b897-e19d-61ba-662deded8c0f/data/data5/current/BP-1141445696-172.17.0.3-1733404627457 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T13:17:22,435 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/cluster_75ee48d7-b897-e19d-61ba-662deded8c0f/data/data6/current/BP-1141445696-172.17.0.3-1733404627457 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T13:17:22,436 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T13:17:22,442 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@395ff33a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T13:17:22,443 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4d24ac65{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T13:17:22,443 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T13:17:22,443 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6331ed3f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T13:17:22,443 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2ff36c4c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/hadoop.log.dir/,STOPPED} 2024-12-05T13:17:22,444 WARN [BP-1141445696-172.17.0.3-1733404627457 heartbeating to localhost/127.0.0.1:43169 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T13:17:22,444 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T13:17:22,444 WARN [BP-1141445696-172.17.0.3-1733404627457 heartbeating to localhost/127.0.0.1:43169 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1141445696-172.17.0.3-1733404627457 (Datanode Uuid 2d5b7502-29e4-4247-91f9-ca32ae6a5af6) service to localhost/127.0.0.1:43169 2024-12-05T13:17:22,444 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T13:17:22,445 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/cluster_75ee48d7-b897-e19d-61ba-662deded8c0f/data/data3/current/BP-1141445696-172.17.0.3-1733404627457 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T13:17:22,445 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/cluster_75ee48d7-b897-e19d-61ba-662deded8c0f/data/data4/current/BP-1141445696-172.17.0.3-1733404627457 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T13:17:22,445 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T13:17:22,450 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@65e17246{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T13:17:22,451 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@220c8d80{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T13:17:22,451 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T13:17:22,451 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@c7135d9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T13:17:22,451 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@51c39823{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/hadoop.log.dir/,STOPPED} 2024-12-05T13:17:22,452 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T13:17:22,452 WARN [BP-1141445696-172.17.0.3-1733404627457 heartbeating to localhost/127.0.0.1:43169 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T13:17:22,452 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T13:17:22,452 WARN [BP-1141445696-172.17.0.3-1733404627457 heartbeating to localhost/127.0.0.1:43169 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1141445696-172.17.0.3-1733404627457 (Datanode Uuid ab30316d-c6dc-4bb6-a173-454b37469f43) service to localhost/127.0.0.1:43169 2024-12-05T13:17:22,453 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/cluster_75ee48d7-b897-e19d-61ba-662deded8c0f/data/data1/current/BP-1141445696-172.17.0.3-1733404627457 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T13:17:22,453 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/cluster_75ee48d7-b897-e19d-61ba-662deded8c0f/data/data2/current/BP-1141445696-172.17.0.3-1733404627457 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T13:17:22,454 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T13:17:22,461 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64418798{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T13:17:22,462 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T13:17:22,462 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T13:17:22,462 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T13:17:22,462 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b615f141-10a3-055d-d7e6-b45e06f674e8/hadoop.log.dir/,STOPPED} 2024-12-05T13:17:22,473 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-05T13:17:22,518 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down