2024-11-13 05:55:08,493 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-13 05:55:08,511 main DEBUG Took 0.016267 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 05:55:08,512 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 05:55:08,512 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 05:55:08,513 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 05:55:08,514 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:55:08,522 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 05:55:08,537 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:55:08,538 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:55:08,539 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:55:08,540 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:55:08,541 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:55:08,541 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:55:08,543 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:55:08,543 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:55:08,544 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:55:08,544 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:55:08,545 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:55:08,546 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:55:08,546 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:55:08,547 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:55:08,547 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:55:08,548 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:55:08,549 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:55:08,549 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:55:08,550 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:55:08,550 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:55:08,551 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:55:08,551 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:55:08,552 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:55:08,552 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:55:08,553 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:55:08,553 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 05:55:08,555 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:55:08,556 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 05:55:08,558 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 05:55:08,559 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 05:55:08,561 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 05:55:08,561 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 05:55:08,572 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 05:55:08,575 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 05:55:08,577 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 05:55:08,578 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 05:55:08,578 main DEBUG createAppenders(={Console}) 2024-11-13 05:55:08,579 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-13 05:55:08,579 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-13 05:55:08,580 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-13 05:55:08,581 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 05:55:08,581 main DEBUG OutputStream closed 2024-11-13 05:55:08,581 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 05:55:08,582 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 05:55:08,582 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-13 05:55:08,669 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 05:55:08,671 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 05:55:08,672 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 05:55:08,673 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 05:55:08,674 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 05:55:08,674 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 05:55:08,675 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 05:55:08,675 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 05:55:08,676 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 05:55:08,676 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 05:55:08,676 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 05:55:08,677 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 05:55:08,677 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 05:55:08,678 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 05:55:08,678 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 05:55:08,678 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 05:55:08,679 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 05:55:08,679 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 05:55:08,681 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 05:55:08,681 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-13 05:55:08,681 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 05:55:08,682 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-13T05:55:08,693 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta timeout: 13 mins 2024-11-13 05:55:08,696 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 05:55:08,696 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T05:55:08,731 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestSCPWithMeta#testRecoveryAndDoubleExecutionOnRsWithMeta Thread=11, OpenFileDescriptor=285, MaxFileDescriptor=1048576, SystemLoadAverage=280, ProcessCount=11, AvailableMemoryMB=8320 2024-11-13T05:55:08,964 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f 2024-11-13T05:55:08,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T05:55:08,980 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/cluster_231b4c26-e46c-eae9-a748-e1bc31753d3a, deleteOnExit=true 2024-11-13T05:55:08,981 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T05:55:08,982 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/test.cache.data in system properties and HBase conf 2024-11-13T05:55:08,982 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T05:55:08,983 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/hadoop.log.dir in system properties and HBase conf 2024-11-13T05:55:08,984 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T05:55:08,984 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T05:55:08,985 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T05:55:09,063 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T05:55:09,153 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:55:09,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T05:55:09,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T05:55:09,158 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T05:55:09,159 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T05:55:09,159 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T05:55:09,160 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T05:55:09,160 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T05:55:09,161 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T05:55:09,161 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T05:55:09,162 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/nfs.dump.dir in system properties and HBase conf 2024-11-13T05:55:09,162 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/java.io.tmpdir in system properties and HBase conf 2024-11-13T05:55:09,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T05:55:09,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T05:55:09,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T05:55:10,120 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T05:55:10,191 INFO [Time-limited test {}] log.Log(170): Logging initialized @2368ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T05:55:10,257 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:55:10,317 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:55:10,341 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:55:10,341 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:55:10,343 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T05:55:10,358 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:55:10,361 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2c447bb0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:55:10,362 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4974766f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:55:10,531 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5183f126{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/java.io.tmpdir/jetty-localhost-44017-hadoop-hdfs-3_4_1-tests_jar-_-any-8839234071157010510/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T05:55:10,538 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4350931a{HTTP/1.1, (http/1.1)}{localhost:44017} 2024-11-13T05:55:10,538 INFO [Time-limited test {}] server.Server(415): Started @2716ms 2024-11-13T05:55:11,029 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:55:11,037 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:55:11,038 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:55:11,038 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:55:11,039 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:55:11,039 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7da31712{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:55:11,040 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@664a7783{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:55:11,143 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4d81bf0e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/java.io.tmpdir/jetty-localhost-38367-hadoop-hdfs-3_4_1-tests_jar-_-any-542161460438044320/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:55:11,144 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@572fe942{HTTP/1.1, (http/1.1)}{localhost:38367} 2024-11-13T05:55:11,144 INFO [Time-limited test {}] server.Server(415): Started @3322ms 2024-11-13T05:55:11,196 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:55:11,297 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:55:11,304 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:55:11,306 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:55:11,307 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:55:11,307 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:55:11,308 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f4017e5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:55:11,309 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6c516a28{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:55:11,434 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1ba455be{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/java.io.tmpdir/jetty-localhost-45519-hadoop-hdfs-3_4_1-tests_jar-_-any-16510291968686569450/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:55:11,435 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@259771a6{HTTP/1.1, (http/1.1)}{localhost:45519} 2024-11-13T05:55:11,435 INFO [Time-limited test {}] server.Server(415): Started @3613ms 2024-11-13T05:55:11,438 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:55:11,471 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T05:55:11,476 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T05:55:11,479 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T05:55:11,479 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T05:55:11,479 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T05:55:11,480 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@30067af9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/hadoop.log.dir/,AVAILABLE} 2024-11-13T05:55:11,481 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@17888407{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T05:55:11,576 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@22333f25{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/java.io.tmpdir/jetty-localhost-46253-hadoop-hdfs-3_4_1-tests_jar-_-any-15323101295914502468/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:55:11,577 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@eec57c1{HTTP/1.1, (http/1.1)}{localhost:46253} 2024-11-13T05:55:11,577 INFO [Time-limited test {}] server.Server(415): Started @3755ms 2024-11-13T05:55:11,579 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T05:55:12,562 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/cluster_231b4c26-e46c-eae9-a748-e1bc31753d3a/data/data1/current/BP-1769423020-172.17.0.2-1731477309687/current, will proceed with Du for space computation calculation, 2024-11-13T05:55:12,562 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/cluster_231b4c26-e46c-eae9-a748-e1bc31753d3a/data/data3/current/BP-1769423020-172.17.0.2-1731477309687/current, will proceed with Du for space computation calculation, 2024-11-13T05:55:12,562 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/cluster_231b4c26-e46c-eae9-a748-e1bc31753d3a/data/data2/current/BP-1769423020-172.17.0.2-1731477309687/current, will proceed with Du for space computation calculation, 2024-11-13T05:55:12,562 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/cluster_231b4c26-e46c-eae9-a748-e1bc31753d3a/data/data4/current/BP-1769423020-172.17.0.2-1731477309687/current, will proceed with Du for space computation calculation, 2024-11-13T05:55:12,597 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:55:12,597 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:55:12,632 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/cluster_231b4c26-e46c-eae9-a748-e1bc31753d3a/data/data5/current/BP-1769423020-172.17.0.2-1731477309687/current, will proceed with Du for space computation calculation, 2024-11-13T05:55:12,632 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/cluster_231b4c26-e46c-eae9-a748-e1bc31753d3a/data/data6/current/BP-1769423020-172.17.0.2-1731477309687/current, will proceed with Du for space computation calculation, 2024-11-13T05:55:12,642 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe9251d4aeb384029 with lease ID 0xf4845b3e338dbb19: Processing first storage report for DS-b1ba899f-55cb-4657-847b-885d11a94025 from datanode DatanodeRegistration(127.0.0.1:41907, datanodeUuid=999c1d0c-5cd8-4a6b-b1fa-b5872fc5c220, infoPort=37481, infoSecurePort=0, ipcPort=35887, storageInfo=lv=-57;cid=testClusterID;nsid=132331298;c=1731477309687) 2024-11-13T05:55:12,644 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe9251d4aeb384029 with lease ID 0xf4845b3e338dbb19: from storage DS-b1ba899f-55cb-4657-847b-885d11a94025 node DatanodeRegistration(127.0.0.1:41907, datanodeUuid=999c1d0c-5cd8-4a6b-b1fa-b5872fc5c220, infoPort=37481, infoSecurePort=0, ipcPort=35887, storageInfo=lv=-57;cid=testClusterID;nsid=132331298;c=1731477309687), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T05:55:12,644 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbbc8ea955b7f6e11 with lease ID 0xf4845b3e338dbb1a: Processing first storage report for DS-f31c5c41-3c6c-4292-9e70-bf8ba9f019f7 from datanode DatanodeRegistration(127.0.0.1:37319, datanodeUuid=50dd1970-68b2-42ac-ad88-d3fbf24e3d13, infoPort=39447, infoSecurePort=0, ipcPort=46199, storageInfo=lv=-57;cid=testClusterID;nsid=132331298;c=1731477309687) 2024-11-13T05:55:12,645 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbbc8ea955b7f6e11 with lease ID 0xf4845b3e338dbb1a: from storage DS-f31c5c41-3c6c-4292-9e70-bf8ba9f019f7 node DatanodeRegistration(127.0.0.1:37319, datanodeUuid=50dd1970-68b2-42ac-ad88-d3fbf24e3d13, infoPort=39447, infoSecurePort=0, ipcPort=46199, storageInfo=lv=-57;cid=testClusterID;nsid=132331298;c=1731477309687), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:55:12,645 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe9251d4aeb384029 with lease ID 0xf4845b3e338dbb19: Processing first storage report for DS-492df007-822b-49cc-b06f-8f78368d05da from datanode DatanodeRegistration(127.0.0.1:41907, datanodeUuid=999c1d0c-5cd8-4a6b-b1fa-b5872fc5c220, infoPort=37481, infoSecurePort=0, ipcPort=35887, storageInfo=lv=-57;cid=testClusterID;nsid=132331298;c=1731477309687) 2024-11-13T05:55:12,645 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe9251d4aeb384029 with lease ID 0xf4845b3e338dbb19: from storage DS-492df007-822b-49cc-b06f-8f78368d05da node DatanodeRegistration(127.0.0.1:41907, datanodeUuid=999c1d0c-5cd8-4a6b-b1fa-b5872fc5c220, infoPort=37481, infoSecurePort=0, ipcPort=35887, storageInfo=lv=-57;cid=testClusterID;nsid=132331298;c=1731477309687), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:55:12,645 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbbc8ea955b7f6e11 with lease ID 0xf4845b3e338dbb1a: Processing first storage report for DS-ebbe7d05-61b9-40b1-a25a-2843fd8ef787 from datanode DatanodeRegistration(127.0.0.1:37319, datanodeUuid=50dd1970-68b2-42ac-ad88-d3fbf24e3d13, infoPort=39447, infoSecurePort=0, ipcPort=46199, storageInfo=lv=-57;cid=testClusterID;nsid=132331298;c=1731477309687) 2024-11-13T05:55:12,646 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbbc8ea955b7f6e11 with lease ID 0xf4845b3e338dbb1a: from storage DS-ebbe7d05-61b9-40b1-a25a-2843fd8ef787 node DatanodeRegistration(127.0.0.1:37319, datanodeUuid=50dd1970-68b2-42ac-ad88-d3fbf24e3d13, infoPort=39447, infoSecurePort=0, ipcPort=46199, storageInfo=lv=-57;cid=testClusterID;nsid=132331298;c=1731477309687), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:55:12,653 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T05:55:12,658 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5a8e75c415c0d748 with lease ID 0xf4845b3e338dbb1b: Processing first storage report for DS-ffc916fc-f85e-4220-8961-748ff0692398 from datanode DatanodeRegistration(127.0.0.1:35961, datanodeUuid=353a8d27-bc02-479e-af6a-5157f17bcd56, infoPort=46875, infoSecurePort=0, ipcPort=41495, storageInfo=lv=-57;cid=testClusterID;nsid=132331298;c=1731477309687) 2024-11-13T05:55:12,658 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5a8e75c415c0d748 with lease ID 0xf4845b3e338dbb1b: from storage DS-ffc916fc-f85e-4220-8961-748ff0692398 node DatanodeRegistration(127.0.0.1:35961, datanodeUuid=353a8d27-bc02-479e-af6a-5157f17bcd56, infoPort=46875, infoSecurePort=0, ipcPort=41495, storageInfo=lv=-57;cid=testClusterID;nsid=132331298;c=1731477309687), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:55:12,659 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5a8e75c415c0d748 with lease ID 0xf4845b3e338dbb1b: Processing first storage report for DS-3483c55d-f02a-4317-9426-3457039ba984 from datanode DatanodeRegistration(127.0.0.1:35961, datanodeUuid=353a8d27-bc02-479e-af6a-5157f17bcd56, infoPort=46875, infoSecurePort=0, ipcPort=41495, storageInfo=lv=-57;cid=testClusterID;nsid=132331298;c=1731477309687) 2024-11-13T05:55:12,659 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5a8e75c415c0d748 with lease ID 0xf4845b3e338dbb1b: from storage DS-3483c55d-f02a-4317-9426-3457039ba984 node DatanodeRegistration(127.0.0.1:35961, datanodeUuid=353a8d27-bc02-479e-af6a-5157f17bcd56, infoPort=46875, infoSecurePort=0, ipcPort=41495, storageInfo=lv=-57;cid=testClusterID;nsid=132331298;c=1731477309687), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T05:55:12,694 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f 2024-11-13T05:55:12,757 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/cluster_231b4c26-e46c-eae9-a748-e1bc31753d3a/zookeeper_0, clientPort=58994, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/cluster_231b4c26-e46c-eae9-a748-e1bc31753d3a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/cluster_231b4c26-e46c-eae9-a748-e1bc31753d3a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T05:55:12,766 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58994 2024-11-13T05:55:12,780 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:55:12,784 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:55:13,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:55:13,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:55:13,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741825_1001 (size=7) 2024-11-13T05:55:13,419 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 with version=8 2024-11-13T05:55:13,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/hbase-staging 2024-11-13T05:55:13,499 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T05:55:13,749 INFO [Time-limited test {}] client.ConnectionUtils(128): master/bc386b7bf2fe:0 server-side Connection retries=45 2024-11-13T05:55:13,757 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:55:13,757 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:55:13,761 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:55:13,762 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:55:13,762 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:55:13,883 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T05:55:13,939 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T05:55:13,948 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T05:55:13,951 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:55:13,974 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 46537 (auto-detected) 2024-11-13T05:55:13,975 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-13T05:55:13,992 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37807 2024-11-13T05:55:14,011 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37807 connecting to ZooKeeper ensemble=127.0.0.1:58994 2024-11-13T05:55:14,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:378070x0, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:55:14,126 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37807-0x101340ca7aa0000 connected 2024-11-13T05:55:14,219 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:55:14,227 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:55:14,238 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:55:14,241 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455, hbase.cluster.distributed=false 2024-11-13T05:55:14,263 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:55:14,267 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37807 2024-11-13T05:55:14,267 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37807 2024-11-13T05:55:14,268 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37807 2024-11-13T05:55:14,268 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37807 2024-11-13T05:55:14,268 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37807 2024-11-13T05:55:14,364 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bc386b7bf2fe:0 server-side Connection retries=45 2024-11-13T05:55:14,365 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:55:14,365 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:55:14,365 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:55:14,366 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:55:14,366 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:55:14,368 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T05:55:14,371 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:55:14,372 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38823 2024-11-13T05:55:14,374 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38823 connecting to ZooKeeper ensemble=127.0.0.1:58994 2024-11-13T05:55:14,375 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:55:14,377 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:55:14,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:388230x0, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:55:14,406 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38823-0x101340ca7aa0001 connected 2024-11-13T05:55:14,407 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38823-0x101340ca7aa0001, quorum=127.0.0.1:58994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:55:14,413 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T05:55:14,421 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T05:55:14,424 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38823-0x101340ca7aa0001, quorum=127.0.0.1:58994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T05:55:14,429 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38823-0x101340ca7aa0001, quorum=127.0.0.1:58994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:55:14,429 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38823 2024-11-13T05:55:14,430 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38823 2024-11-13T05:55:14,430 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38823 2024-11-13T05:55:14,432 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38823 2024-11-13T05:55:14,432 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38823 2024-11-13T05:55:14,448 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bc386b7bf2fe:0 server-side Connection retries=45 2024-11-13T05:55:14,449 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:55:14,449 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:55:14,449 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:55:14,449 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:55:14,450 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:55:14,450 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T05:55:14,450 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:55:14,451 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45383 2024-11-13T05:55:14,453 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45383 connecting to ZooKeeper ensemble=127.0.0.1:58994 2024-11-13T05:55:14,454 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:55:14,456 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:55:14,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:453830x0, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:55:14,474 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:453830x0, quorum=127.0.0.1:58994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:55:14,474 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45383-0x101340ca7aa0002 connected 2024-11-13T05:55:14,475 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T05:55:14,476 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T05:55:14,477 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T05:55:14,479 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:55:14,479 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45383 2024-11-13T05:55:14,479 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45383 2024-11-13T05:55:14,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45383 2024-11-13T05:55:14,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45383 2024-11-13T05:55:14,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45383 2024-11-13T05:55:14,500 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bc386b7bf2fe:0 server-side Connection retries=45 2024-11-13T05:55:14,500 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:55:14,500 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:55:14,500 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:55:14,501 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:55:14,501 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:55:14,501 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T05:55:14,501 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:55:14,502 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40327 2024-11-13T05:55:14,505 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40327 connecting to ZooKeeper ensemble=127.0.0.1:58994 2024-11-13T05:55:14,506 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:55:14,509 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:55:14,523 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:403270x0, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:55:14,524 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40327-0x101340ca7aa0003 connected 2024-11-13T05:55:14,524 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:55:14,525 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T05:55:14,527 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T05:55:14,528 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T05:55:14,530 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:55:14,531 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40327 2024-11-13T05:55:14,531 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40327 2024-11-13T05:55:14,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40327 2024-11-13T05:55:14,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40327 2024-11-13T05:55:14,534 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40327 2024-11-13T05:55:14,551 DEBUG [M:0;bc386b7bf2fe:37807 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;bc386b7bf2fe:37807 2024-11-13T05:55:14,552 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/bc386b7bf2fe,37807,1731477313603 2024-11-13T05:55:14,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38823-0x101340ca7aa0001, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:55:14,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:55:14,565 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:55:14,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:55:14,568 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/bc386b7bf2fe,37807,1731477313603 2024-11-13T05:55:14,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38823-0x101340ca7aa0001, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T05:55:14,597 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T05:55:14,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T05:55:14,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:14,597 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:14,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38823-0x101340ca7aa0001, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:14,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:14,598 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T05:55:14,600 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/bc386b7bf2fe,37807,1731477313603 from backup master directory 2024-11-13T05:55:14,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/bc386b7bf2fe,37807,1731477313603 2024-11-13T05:55:14,606 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:55:14,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38823-0x101340ca7aa0001, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:55:14,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:55:14,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:55:14,607 WARN [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:55:14,607 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=bc386b7bf2fe,37807,1731477313603 2024-11-13T05:55:14,609 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T05:55:14,610 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T05:55:14,665 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/hbase.id] with ID: 8e8c3ff9-23cc-4c45-b323-781f8a3738fe 2024-11-13T05:55:14,665 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/.tmp/hbase.id 2024-11-13T05:55:14,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741826_1002 (size=42) 2024-11-13T05:55:14,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741826_1002 (size=42) 2024-11-13T05:55:14,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741826_1002 (size=42) 2024-11-13T05:55:14,682 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/.tmp/hbase.id]:[hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/hbase.id] 2024-11-13T05:55:14,728 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T05:55:14,732 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T05:55:14,749 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-13T05:55:14,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:14,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38823-0x101340ca7aa0001, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:14,772 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:14,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:14,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:55:14,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:55:14,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741827_1003 (size=196) 2024-11-13T05:55:14,805 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:55:14,807 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T05:55:14,820 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:55:14,823 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:55:14,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741828_1004 (size=1189) 2024-11-13T05:55:14,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741828_1004 (size=1189) 2024-11-13T05:55:14,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741828_1004 (size=1189) 2024-11-13T05:55:14,867 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/data/master/store 2024-11-13T05:55:14,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:55:14,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:55:14,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741829_1005 (size=34) 2024-11-13T05:55:14,893 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T05:55:14,896 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:14,897 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:55:14,897 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:55:14,898 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:55:14,899 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:55:14,899 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:55:14,899 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:55:14,900 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731477314897Disabling compacts and flushes for region at 1731477314897Disabling writes for close at 1731477314899 (+2 ms)Writing region close event to WAL at 1731477314899Closed at 1731477314899 2024-11-13T05:55:14,903 WARN [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/data/master/store/.initializing 2024-11-13T05:55:14,903 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/WALs/bc386b7bf2fe,37807,1731477313603 2024-11-13T05:55:14,910 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:55:14,923 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bc386b7bf2fe%2C37807%2C1731477313603, suffix=, logDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/WALs/bc386b7bf2fe,37807,1731477313603, archiveDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/oldWALs, maxLogs=10 2024-11-13T05:55:14,943 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/WALs/bc386b7bf2fe,37807,1731477313603/bc386b7bf2fe%2C37807%2C1731477313603.1731477314927, exclude list is [], retry=0 2024-11-13T05:55:14,961 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35961,DS-ffc916fc-f85e-4220-8961-748ff0692398,DISK] 2024-11-13T05:55:14,961 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37319,DS-f31c5c41-3c6c-4292-9e70-bf8ba9f019f7,DISK] 2024-11-13T05:55:14,961 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41907,DS-b1ba899f-55cb-4657-847b-885d11a94025,DISK] 2024-11-13T05:55:14,964 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T05:55:14,999 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/WALs/bc386b7bf2fe,37807,1731477313603/bc386b7bf2fe%2C37807%2C1731477313603.1731477314927 2024-11-13T05:55:15,000 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39447:39447),(127.0.0.1/127.0.0.1:37481:37481),(127.0.0.1/127.0.0.1:46875:46875)] 2024-11-13T05:55:15,000 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:55:15,001 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:15,004 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:55:15,005 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:55:15,039 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:55:15,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T05:55:15,064 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:15,066 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:55:15,067 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:55:15,070 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T05:55:15,070 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:15,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:15,072 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:55:15,075 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T05:55:15,075 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:15,076 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:15,077 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:55:15,080 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T05:55:15,080 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:15,081 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:15,081 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:55:15,084 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:55:15,086 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:55:15,090 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:55:15,091 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:55:15,094 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:55:15,098 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:55:15,103 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:15,105 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74419748, jitterRate=0.10894066095352173}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:55:15,116 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731477315016Initializing all the Stores at 1731477315018 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477315019 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477315020 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477315020Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477315020Cleaning up temporary data from old regions at 1731477315091 (+71 ms)Region opened successfully at 1731477315116 (+25 ms) 2024-11-13T05:55:15,118 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T05:55:15,150 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e920141, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bc386b7bf2fe/172.17.0.2:0 2024-11-13T05:55:15,179 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T05:55:15,188 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T05:55:15,188 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T05:55:15,190 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T05:55:15,192 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T05:55:15,198 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-13T05:55:15,198 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T05:55:15,221 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T05:55:15,230 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T05:55:15,265 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T05:55:15,269 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T05:55:15,272 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T05:55:15,281 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T05:55:15,284 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T05:55:15,289 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T05:55:15,298 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T05:55:15,301 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T05:55:15,306 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T05:55:15,328 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T05:55:15,338 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T05:55:15,348 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:55:15,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:55:15,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38823-0x101340ca7aa0001, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:55:15,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:55:15,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:15,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:15,348 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:15,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38823-0x101340ca7aa0001, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:15,353 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=bc386b7bf2fe,37807,1731477313603, sessionid=0x101340ca7aa0000, setting cluster-up flag (Was=false) 2024-11-13T05:55:15,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:15,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:15,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38823-0x101340ca7aa0001, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:15,382 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:15,407 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T05:55:15,411 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bc386b7bf2fe,37807,1731477313603 2024-11-13T05:55:15,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:15,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:15,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38823-0x101340ca7aa0001, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:15,432 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:15,456 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T05:55:15,458 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bc386b7bf2fe,37807,1731477313603 2024-11-13T05:55:15,466 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T05:55:15,536 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T05:55:15,540 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(746): ClusterId : 8e8c3ff9-23cc-4c45-b323-781f8a3738fe 2024-11-13T05:55:15,540 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(746): ClusterId : 8e8c3ff9-23cc-4c45-b323-781f8a3738fe 2024-11-13T05:55:15,540 INFO [RS:0;bc386b7bf2fe:38823 {}] regionserver.HRegionServer(746): ClusterId : 8e8c3ff9-23cc-4c45-b323-781f8a3738fe 2024-11-13T05:55:15,543 DEBUG [RS:0;bc386b7bf2fe:38823 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T05:55:15,543 DEBUG [RS:2;bc386b7bf2fe:40327 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T05:55:15,543 DEBUG [RS:1;bc386b7bf2fe:45383 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T05:55:15,546 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T05:55:15,552 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T05:55:15,557 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: bc386b7bf2fe,37807,1731477313603 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T05:55:15,575 DEBUG [RS:0;bc386b7bf2fe:38823 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T05:55:15,575 DEBUG [RS:2;bc386b7bf2fe:40327 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T05:55:15,575 DEBUG [RS:1;bc386b7bf2fe:45383 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T05:55:15,575 DEBUG [RS:0;bc386b7bf2fe:38823 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T05:55:15,575 DEBUG [RS:1;bc386b7bf2fe:45383 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T05:55:15,575 DEBUG [RS:2;bc386b7bf2fe:40327 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T05:55:15,575 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:55:15,576 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:55:15,576 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:55:15,576 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:55:15,576 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/bc386b7bf2fe:0, corePoolSize=10, maxPoolSize=10 2024-11-13T05:55:15,576 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,576 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:55:15,576 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,577 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731477345577 2024-11-13T05:55:15,579 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T05:55:15,580 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T05:55:15,582 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:55:15,582 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T05:55:15,583 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T05:55:15,583 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T05:55:15,584 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T05:55:15,584 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T05:55:15,585 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,588 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:15,588 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T05:55:15,588 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T05:55:15,589 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T05:55:15,590 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T05:55:15,590 DEBUG [RS:0;bc386b7bf2fe:38823 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T05:55:15,590 DEBUG [RS:1;bc386b7bf2fe:45383 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T05:55:15,591 DEBUG [RS:2;bc386b7bf2fe:40327 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T05:55:15,591 DEBUG [RS:2;bc386b7bf2fe:40327 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68fd8704, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bc386b7bf2fe/172.17.0.2:0 2024-11-13T05:55:15,591 DEBUG [RS:0;bc386b7bf2fe:38823 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c001ffa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bc386b7bf2fe/172.17.0.2:0 2024-11-13T05:55:15,591 DEBUG [RS:1;bc386b7bf2fe:45383 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fda22b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bc386b7bf2fe/172.17.0.2:0 2024-11-13T05:55:15,594 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T05:55:15,595 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T05:55:15,599 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.large.0-1731477315596,5,FailOnTimeoutGroup] 2024-11-13T05:55:15,603 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.small.0-1731477315599,5,FailOnTimeoutGroup] 2024-11-13T05:55:15,603 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,603 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T05:55:15,605 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,605 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,609 DEBUG [RS:1;bc386b7bf2fe:45383 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;bc386b7bf2fe:45383 2024-11-13T05:55:15,611 DEBUG [RS:2;bc386b7bf2fe:40327 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;bc386b7bf2fe:40327 2024-11-13T05:55:15,611 DEBUG [RS:0;bc386b7bf2fe:38823 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;bc386b7bf2fe:38823 2024-11-13T05:55:15,612 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T05:55:15,612 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T05:55:15,612 INFO [RS:0;bc386b7bf2fe:38823 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T05:55:15,612 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T05:55:15,612 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T05:55:15,612 INFO [RS:0;bc386b7bf2fe:38823 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T05:55:15,612 DEBUG [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T05:55:15,612 DEBUG [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T05:55:15,612 DEBUG [RS:0;bc386b7bf2fe:38823 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T05:55:15,616 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(2659): reportForDuty to master=bc386b7bf2fe,37807,1731477313603 with port=40327, startcode=1731477314499 2024-11-13T05:55:15,616 INFO [RS:0;bc386b7bf2fe:38823 {}] regionserver.HRegionServer(2659): reportForDuty to master=bc386b7bf2fe,37807,1731477313603 with port=38823, startcode=1731477314332 2024-11-13T05:55:15,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741831_1007 (size=1321) 2024-11-13T05:55:15,616 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(2659): reportForDuty to master=bc386b7bf2fe,37807,1731477313603 with port=45383, startcode=1731477314448 2024-11-13T05:55:15,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741831_1007 (size=1321) 2024-11-13T05:55:15,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741831_1007 (size=1321) 2024-11-13T05:55:15,619 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T05:55:15,620 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:15,631 DEBUG [RS:0;bc386b7bf2fe:38823 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T05:55:15,631 DEBUG [RS:2;bc386b7bf2fe:40327 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T05:55:15,631 DEBUG [RS:1;bc386b7bf2fe:45383 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T05:55:15,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741832_1008 (size=32) 2024-11-13T05:55:15,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741832_1008 (size=32) 2024-11-13T05:55:15,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741832_1008 (size=32) 2024-11-13T05:55:15,649 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:15,652 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:55:15,657 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:55:15,658 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:15,659 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:55:15,659 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:55:15,662 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:55:15,663 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:15,664 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:55:15,664 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:55:15,667 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:55:15,667 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:15,668 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46071, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T05:55:15,668 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38391, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T05:55:15,668 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:55:15,668 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:55:15,671 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:55:15,671 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:15,673 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35961, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T05:55:15,673 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:55:15,673 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:55:15,674 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740 2024-11-13T05:55:15,675 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740 2024-11-13T05:55:15,676 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:15,678 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:55:15,678 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:55:15,679 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] master.ServerManager(517): Registering regionserver=bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:15,679 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:55:15,682 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:55:15,687 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:15,688 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59981475, jitterRate=-0.10620637238025665}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:55:15,690 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:15,690 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] master.ServerManager(517): Registering regionserver=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:15,694 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bc386b7bf2fe,40327,1731477314499 2024-11-13T05:55:15,694 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] master.ServerManager(517): Registering regionserver=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:55:15,694 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731477315649Initializing all the Stores at 1731477315651 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477315651Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477315652 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477315652Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477315652Cleaning up temporary data from old regions at 1731477315678 (+26 ms)Region opened successfully at 1731477315694 (+16 ms) 2024-11-13T05:55:15,694 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:55:15,694 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:55:15,694 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:55:15,694 DEBUG [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:15,694 DEBUG [RS:0;bc386b7bf2fe:38823 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:15,694 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:55:15,694 DEBUG [RS:0;bc386b7bf2fe:38823 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37737 2024-11-13T05:55:15,694 DEBUG [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37737 2024-11-13T05:55:15,694 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:55:15,694 DEBUG [RS:0;bc386b7bf2fe:38823 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T05:55:15,694 DEBUG [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T05:55:15,696 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:55:15,697 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731477315694Disabling compacts and flushes for region at 1731477315694Disabling writes for close at 1731477315694Writing region close event to WAL at 1731477315696 (+2 ms)Closed at 1731477315696 2024-11-13T05:55:15,698 DEBUG [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:15,698 DEBUG [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37737 2024-11-13T05:55:15,698 DEBUG [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T05:55:15,700 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:55:15,701 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T05:55:15,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T05:55:15,719 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:55:15,721 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T05:55:15,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:55:15,773 DEBUG [RS:0;bc386b7bf2fe:38823 {}] zookeeper.ZKUtil(111): regionserver:38823-0x101340ca7aa0001, quorum=127.0.0.1:58994, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:15,773 DEBUG [RS:1;bc386b7bf2fe:45383 {}] zookeeper.ZKUtil(111): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:15,774 WARN [RS:1;bc386b7bf2fe:45383 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:55:15,774 WARN [RS:0;bc386b7bf2fe:38823 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:55:15,774 INFO [RS:0;bc386b7bf2fe:38823 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:55:15,774 INFO [RS:1;bc386b7bf2fe:45383 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:55:15,775 DEBUG [RS:0;bc386b7bf2fe:38823 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:15,775 DEBUG [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:15,775 DEBUG [RS:2;bc386b7bf2fe:40327 {}] zookeeper.ZKUtil(111): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bc386b7bf2fe,40327,1731477314499 2024-11-13T05:55:15,775 WARN [RS:2;bc386b7bf2fe:40327 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:55:15,776 INFO [RS:2;bc386b7bf2fe:40327 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:55:15,776 DEBUG [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,40327,1731477314499 2024-11-13T05:55:15,777 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bc386b7bf2fe,45383,1731477314448] 2024-11-13T05:55:15,778 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bc386b7bf2fe,40327,1731477314499] 2024-11-13T05:55:15,778 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bc386b7bf2fe,38823,1731477314332] 2024-11-13T05:55:15,800 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T05:55:15,800 INFO [RS:0;bc386b7bf2fe:38823 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T05:55:15,800 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T05:55:15,817 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T05:55:15,817 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T05:55:15,817 INFO [RS:0;bc386b7bf2fe:38823 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T05:55:15,824 INFO [RS:0;bc386b7bf2fe:38823 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:55:15,824 INFO [RS:2;bc386b7bf2fe:40327 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:55:15,824 INFO [RS:1;bc386b7bf2fe:45383 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:55:15,824 INFO [RS:0;bc386b7bf2fe:38823 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,824 INFO [RS:1;bc386b7bf2fe:45383 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,824 INFO [RS:2;bc386b7bf2fe:40327 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,825 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T05:55:15,825 INFO [RS:0;bc386b7bf2fe:38823 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T05:55:15,825 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T05:55:15,831 INFO [RS:1;bc386b7bf2fe:45383 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T05:55:15,831 INFO [RS:2;bc386b7bf2fe:40327 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T05:55:15,831 INFO [RS:0;bc386b7bf2fe:38823 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T05:55:15,832 INFO [RS:1;bc386b7bf2fe:45383 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,832 INFO [RS:0;bc386b7bf2fe:38823 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,832 INFO [RS:2;bc386b7bf2fe:40327 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,833 DEBUG [RS:1;bc386b7bf2fe:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,833 DEBUG [RS:2;bc386b7bf2fe:40327 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,833 DEBUG [RS:0;bc386b7bf2fe:38823 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,833 DEBUG [RS:1;bc386b7bf2fe:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,833 DEBUG [RS:2;bc386b7bf2fe:40327 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,833 DEBUG [RS:0;bc386b7bf2fe:38823 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,833 DEBUG [RS:1;bc386b7bf2fe:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,833 DEBUG [RS:0;bc386b7bf2fe:38823 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,833 DEBUG [RS:2;bc386b7bf2fe:40327 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,833 DEBUG [RS:1;bc386b7bf2fe:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,833 DEBUG [RS:2;bc386b7bf2fe:40327 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,833 DEBUG [RS:0;bc386b7bf2fe:38823 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,833 DEBUG [RS:1;bc386b7bf2fe:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,833 DEBUG [RS:2;bc386b7bf2fe:40327 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,833 DEBUG [RS:0;bc386b7bf2fe:38823 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,833 DEBUG [RS:1;bc386b7bf2fe:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:55:15,834 DEBUG [RS:1;bc386b7bf2fe:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,834 DEBUG [RS:0;bc386b7bf2fe:38823 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:55:15,834 DEBUG [RS:2;bc386b7bf2fe:40327 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:55:15,834 DEBUG [RS:1;bc386b7bf2fe:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,834 DEBUG [RS:0;bc386b7bf2fe:38823 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,834 DEBUG [RS:2;bc386b7bf2fe:40327 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,834 DEBUG [RS:1;bc386b7bf2fe:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,834 DEBUG [RS:0;bc386b7bf2fe:38823 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,834 DEBUG [RS:2;bc386b7bf2fe:40327 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,834 DEBUG [RS:1;bc386b7bf2fe:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,834 DEBUG [RS:1;bc386b7bf2fe:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,834 DEBUG [RS:2;bc386b7bf2fe:40327 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,834 DEBUG [RS:0;bc386b7bf2fe:38823 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,834 DEBUG [RS:1;bc386b7bf2fe:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,834 DEBUG [RS:0;bc386b7bf2fe:38823 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,834 DEBUG [RS:2;bc386b7bf2fe:40327 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,834 DEBUG [RS:1;bc386b7bf2fe:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:55:15,834 DEBUG [RS:2;bc386b7bf2fe:40327 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,834 DEBUG [RS:0;bc386b7bf2fe:38823 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,834 DEBUG [RS:1;bc386b7bf2fe:45383 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:55:15,834 DEBUG [RS:0;bc386b7bf2fe:38823 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,834 DEBUG [RS:2;bc386b7bf2fe:40327 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:55:15,835 DEBUG [RS:0;bc386b7bf2fe:38823 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:55:15,835 DEBUG [RS:2;bc386b7bf2fe:40327 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:55:15,835 DEBUG [RS:2;bc386b7bf2fe:40327 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:55:15,835 DEBUG [RS:0;bc386b7bf2fe:38823 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:55:15,838 INFO [RS:1;bc386b7bf2fe:45383 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,838 INFO [RS:0;bc386b7bf2fe:38823 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,839 INFO [RS:0;bc386b7bf2fe:38823 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,839 INFO [RS:1;bc386b7bf2fe:45383 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,839 INFO [RS:0;bc386b7bf2fe:38823 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,839 INFO [RS:1;bc386b7bf2fe:45383 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,839 INFO [RS:0;bc386b7bf2fe:38823 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,839 INFO [RS:1;bc386b7bf2fe:45383 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,839 INFO [RS:0;bc386b7bf2fe:38823 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,839 INFO [RS:0;bc386b7bf2fe:38823 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,38823,1731477314332-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:55:15,839 INFO [RS:1;bc386b7bf2fe:45383 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,839 INFO [RS:1;bc386b7bf2fe:45383 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,45383,1731477314448-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:55:15,839 INFO [RS:2;bc386b7bf2fe:40327 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,839 INFO [RS:2;bc386b7bf2fe:40327 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,839 INFO [RS:2;bc386b7bf2fe:40327 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,839 INFO [RS:2;bc386b7bf2fe:40327 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,840 INFO [RS:2;bc386b7bf2fe:40327 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,840 INFO [RS:2;bc386b7bf2fe:40327 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,40327,1731477314499-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:55:15,862 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T05:55:15,862 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T05:55:15,862 INFO [RS:0;bc386b7bf2fe:38823 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T05:55:15,865 INFO [RS:1;bc386b7bf2fe:45383 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,45383,1731477314448-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,865 INFO [RS:2;bc386b7bf2fe:40327 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,40327,1731477314499-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,865 INFO [RS:0;bc386b7bf2fe:38823 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,38823,1731477314332-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,865 INFO [RS:0;bc386b7bf2fe:38823 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,865 INFO [RS:1;bc386b7bf2fe:45383 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,865 INFO [RS:2;bc386b7bf2fe:40327 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,865 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.Replication(171): bc386b7bf2fe,40327,1731477314499 started 2024-11-13T05:55:15,865 INFO [RS:0;bc386b7bf2fe:38823 {}] regionserver.Replication(171): bc386b7bf2fe,38823,1731477314332 started 2024-11-13T05:55:15,865 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.Replication(171): bc386b7bf2fe,45383,1731477314448 started 2024-11-13T05:55:15,872 WARN [bc386b7bf2fe:37807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-13T05:55:15,889 INFO [RS:1;bc386b7bf2fe:45383 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,889 INFO [RS:0;bc386b7bf2fe:38823 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,889 INFO [RS:2;bc386b7bf2fe:40327 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:15,890 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(1482): Serving as bc386b7bf2fe,45383,1731477314448, RpcServer on bc386b7bf2fe/172.17.0.2:45383, sessionid=0x101340ca7aa0002 2024-11-13T05:55:15,890 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(1482): Serving as bc386b7bf2fe,40327,1731477314499, RpcServer on bc386b7bf2fe/172.17.0.2:40327, sessionid=0x101340ca7aa0003 2024-11-13T05:55:15,890 INFO [RS:0;bc386b7bf2fe:38823 {}] regionserver.HRegionServer(1482): Serving as bc386b7bf2fe,38823,1731477314332, RpcServer on bc386b7bf2fe/172.17.0.2:38823, sessionid=0x101340ca7aa0001 2024-11-13T05:55:15,891 DEBUG [RS:0;bc386b7bf2fe:38823 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T05:55:15,891 DEBUG [RS:1;bc386b7bf2fe:45383 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T05:55:15,891 DEBUG [RS:2;bc386b7bf2fe:40327 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T05:55:15,891 DEBUG [RS:0;bc386b7bf2fe:38823 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:15,891 DEBUG [RS:2;bc386b7bf2fe:40327 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bc386b7bf2fe,40327,1731477314499 2024-11-13T05:55:15,891 DEBUG [RS:1;bc386b7bf2fe:45383 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:15,891 DEBUG [RS:2;bc386b7bf2fe:40327 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,40327,1731477314499' 2024-11-13T05:55:15,891 DEBUG [RS:0;bc386b7bf2fe:38823 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,38823,1731477314332' 2024-11-13T05:55:15,891 DEBUG [RS:1;bc386b7bf2fe:45383 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,45383,1731477314448' 2024-11-13T05:55:15,891 DEBUG [RS:1;bc386b7bf2fe:45383 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T05:55:15,891 DEBUG [RS:2;bc386b7bf2fe:40327 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T05:55:15,891 DEBUG [RS:0;bc386b7bf2fe:38823 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T05:55:15,892 DEBUG [RS:2;bc386b7bf2fe:40327 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T05:55:15,892 DEBUG [RS:0;bc386b7bf2fe:38823 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T05:55:15,892 DEBUG [RS:1;bc386b7bf2fe:45383 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T05:55:15,893 DEBUG [RS:1;bc386b7bf2fe:45383 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T05:55:15,893 DEBUG [RS:2;bc386b7bf2fe:40327 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T05:55:15,893 DEBUG [RS:0;bc386b7bf2fe:38823 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T05:55:15,893 DEBUG [RS:1;bc386b7bf2fe:45383 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T05:55:15,893 DEBUG [RS:2;bc386b7bf2fe:40327 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T05:55:15,893 DEBUG [RS:0;bc386b7bf2fe:38823 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T05:55:15,893 DEBUG [RS:1;bc386b7bf2fe:45383 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:15,893 DEBUG [RS:1;bc386b7bf2fe:45383 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,45383,1731477314448' 2024-11-13T05:55:15,893 DEBUG [RS:2;bc386b7bf2fe:40327 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bc386b7bf2fe,40327,1731477314499 2024-11-13T05:55:15,893 DEBUG [RS:0;bc386b7bf2fe:38823 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:15,893 DEBUG [RS:1;bc386b7bf2fe:45383 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T05:55:15,893 DEBUG [RS:0;bc386b7bf2fe:38823 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,38823,1731477314332' 2024-11-13T05:55:15,893 DEBUG [RS:2;bc386b7bf2fe:40327 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,40327,1731477314499' 2024-11-13T05:55:15,893 DEBUG [RS:2;bc386b7bf2fe:40327 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T05:55:15,893 DEBUG [RS:0;bc386b7bf2fe:38823 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T05:55:15,894 DEBUG [RS:1;bc386b7bf2fe:45383 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T05:55:15,894 DEBUG [RS:2;bc386b7bf2fe:40327 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T05:55:15,894 DEBUG [RS:0;bc386b7bf2fe:38823 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T05:55:15,894 DEBUG [RS:1;bc386b7bf2fe:45383 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T05:55:15,894 DEBUG [RS:2;bc386b7bf2fe:40327 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T05:55:15,894 INFO [RS:1;bc386b7bf2fe:45383 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T05:55:15,895 DEBUG [RS:0;bc386b7bf2fe:38823 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T05:55:15,895 INFO [RS:2;bc386b7bf2fe:40327 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T05:55:15,895 INFO [RS:1;bc386b7bf2fe:45383 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T05:55:15,895 INFO [RS:0;bc386b7bf2fe:38823 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T05:55:15,895 INFO [RS:2;bc386b7bf2fe:40327 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T05:55:15,895 INFO [RS:0;bc386b7bf2fe:38823 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T05:55:16,001 INFO [RS:1;bc386b7bf2fe:45383 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:55:16,002 INFO [RS:2;bc386b7bf2fe:40327 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:55:16,002 INFO [RS:0;bc386b7bf2fe:38823 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:55:16,006 INFO [RS:1;bc386b7bf2fe:45383 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bc386b7bf2fe%2C45383%2C1731477314448, suffix=, logDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,45383,1731477314448, archiveDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/oldWALs, maxLogs=32 2024-11-13T05:55:16,006 INFO [RS:0;bc386b7bf2fe:38823 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bc386b7bf2fe%2C38823%2C1731477314332, suffix=, logDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332, archiveDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/oldWALs, maxLogs=32 2024-11-13T05:55:16,007 INFO [RS:2;bc386b7bf2fe:40327 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bc386b7bf2fe%2C40327%2C1731477314499, suffix=, logDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,40327,1731477314499, archiveDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/oldWALs, maxLogs=32 2024-11-13T05:55:16,023 DEBUG [RS:1;bc386b7bf2fe:45383 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,45383,1731477314448/bc386b7bf2fe%2C45383%2C1731477314448.1731477316011, exclude list is [], retry=0 2024-11-13T05:55:16,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35961,DS-ffc916fc-f85e-4220-8961-748ff0692398,DISK] 2024-11-13T05:55:16,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41907,DS-b1ba899f-55cb-4657-847b-885d11a94025,DISK] 2024-11-13T05:55:16,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37319,DS-f31c5c41-3c6c-4292-9e70-bf8ba9f019f7,DISK] 2024-11-13T05:55:16,030 DEBUG [RS:0;bc386b7bf2fe:38823 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332/bc386b7bf2fe%2C38823%2C1731477314332.1731477316011, exclude list is [], retry=0 2024-11-13T05:55:16,030 DEBUG [RS:2;bc386b7bf2fe:40327 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,40327,1731477314499/bc386b7bf2fe%2C40327%2C1731477314499.1731477316011, exclude list is [], retry=0 2024-11-13T05:55:16,032 INFO [RS:1;bc386b7bf2fe:45383 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,45383,1731477314448/bc386b7bf2fe%2C45383%2C1731477314448.1731477316011 2024-11-13T05:55:16,032 DEBUG [RS:1;bc386b7bf2fe:45383 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39447:39447),(127.0.0.1/127.0.0.1:46875:46875),(127.0.0.1/127.0.0.1:37481:37481)] 2024-11-13T05:55:16,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37319,DS-f31c5c41-3c6c-4292-9e70-bf8ba9f019f7,DISK] 2024-11-13T05:55:16,035 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41907,DS-b1ba899f-55cb-4657-847b-885d11a94025,DISK] 2024-11-13T05:55:16,035 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35961,DS-ffc916fc-f85e-4220-8961-748ff0692398,DISK] 2024-11-13T05:55:16,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41907,DS-b1ba899f-55cb-4657-847b-885d11a94025,DISK] 2024-11-13T05:55:16,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35961,DS-ffc916fc-f85e-4220-8961-748ff0692398,DISK] 2024-11-13T05:55:16,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37319,DS-f31c5c41-3c6c-4292-9e70-bf8ba9f019f7,DISK] 2024-11-13T05:55:16,041 INFO [RS:0;bc386b7bf2fe:38823 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332/bc386b7bf2fe%2C38823%2C1731477314332.1731477316011 2024-11-13T05:55:16,041 DEBUG [RS:0;bc386b7bf2fe:38823 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37481:37481),(127.0.0.1/127.0.0.1:39447:39447),(127.0.0.1/127.0.0.1:46875:46875)] 2024-11-13T05:55:16,042 INFO [RS:2;bc386b7bf2fe:40327 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,40327,1731477314499/bc386b7bf2fe%2C40327%2C1731477314499.1731477316011 2024-11-13T05:55:16,043 DEBUG [RS:2;bc386b7bf2fe:40327 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37481:37481),(127.0.0.1/127.0.0.1:46875:46875),(127.0.0.1/127.0.0.1:39447:39447)] 2024-11-13T05:55:16,127 DEBUG [bc386b7bf2fe:37807 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-13T05:55:16,139 DEBUG [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(204): Hosts are {bc386b7bf2fe=0} racks are {/default-rack=0} 2024-11-13T05:55:16,145 DEBUG [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T05:55:16,145 DEBUG [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T05:55:16,145 DEBUG [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T05:55:16,145 DEBUG [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T05:55:16,145 DEBUG [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T05:55:16,145 DEBUG [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T05:55:16,145 INFO [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T05:55:16,145 INFO [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T05:55:16,145 INFO [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T05:55:16,145 DEBUG [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T05:55:16,153 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:16,159 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,38823,1731477314332, state=OPENING 2024-11-13T05:55:16,215 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T05:55:16,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:16,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38823-0x101340ca7aa0001, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:16,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:16,223 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:55:16,225 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:55:16,225 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:55:16,225 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:55:16,225 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:55:16,228 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:55:16,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,38823,1731477314332}] 2024-11-13T05:55:16,413 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T05:55:16,415 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41847, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:55:16,427 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T05:55:16,427 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:55:16,427 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T05:55:16,430 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bc386b7bf2fe%2C38823%2C1731477314332.meta, suffix=.meta, logDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332, archiveDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/oldWALs, maxLogs=32 2024-11-13T05:55:16,444 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332/bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, exclude list is [], retry=0 2024-11-13T05:55:16,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37319,DS-f31c5c41-3c6c-4292-9e70-bf8ba9f019f7,DISK] 2024-11-13T05:55:16,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35961,DS-ffc916fc-f85e-4220-8961-748ff0692398,DISK] 2024-11-13T05:55:16,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41907,DS-b1ba899f-55cb-4657-847b-885d11a94025,DISK] 2024-11-13T05:55:16,451 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332/bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta 2024-11-13T05:55:16,452 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39447:39447),(127.0.0.1/127.0.0.1:46875:46875),(127.0.0.1/127.0.0.1:37481:37481)] 2024-11-13T05:55:16,452 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:55:16,453 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T05:55:16,455 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T05:55:16,460 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T05:55:16,463 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T05:55:16,464 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:16,464 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T05:55:16,464 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T05:55:16,470 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:55:16,471 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:55:16,472 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:16,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:55:16,473 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:55:16,474 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:55:16,474 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:16,476 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:55:16,476 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:55:16,478 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:55:16,478 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:16,479 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:55:16,479 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:55:16,480 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:55:16,481 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:16,481 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:55:16,482 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:55:16,483 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740 2024-11-13T05:55:16,485 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740 2024-11-13T05:55:16,488 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:55:16,488 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:55:16,489 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:55:16,491 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:55:16,493 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73891560, jitterRate=0.10107004642486572}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:55:16,493 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T05:55:16,494 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731477316467Writing region info on filesystem at 1731477316467Initializing all the Stores at 1731477316469 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477316469Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477316469Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477316469Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477316469Cleaning up temporary data from old regions at 1731477316488 (+19 ms)Running coprocessor post-open hooks at 1731477316493 (+5 ms)Region opened successfully at 1731477316494 (+1 ms) 2024-11-13T05:55:16,501 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731477316402 2024-11-13T05:55:16,510 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T05:55:16,511 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T05:55:16,513 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:16,515 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,38823,1731477314332, state=OPEN 2024-11-13T05:55:16,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38823-0x101340ca7aa0001, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:55:16,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:55:16,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:55:16,547 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:55:16,548 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:55:16,548 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:55:16,548 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:55:16,548 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:55:16,549 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:16,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T05:55:16,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,38823,1731477314332 in 318 msec 2024-11-13T05:55:16,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T05:55:16,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 853 msec 2024-11-13T05:55:16,566 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:55:16,567 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T05:55:16,590 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:55:16,592 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:55:16,617 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:55:16,619 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51135, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:55:16,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1460 sec 2024-11-13T05:55:16,642 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731477316642, completionTime=-1 2024-11-13T05:55:16,644 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-13T05:55:16,644 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T05:55:16,668 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-13T05:55:16,668 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731477376668 2024-11-13T05:55:16,669 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731477436669 2024-11-13T05:55:16,669 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-13T05:55:16,670 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-13T05:55:16,677 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,37807,1731477313603-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:16,677 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,37807,1731477313603-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:16,677 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,37807,1731477313603-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:16,679 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-bc386b7bf2fe:37807, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:16,679 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:16,680 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:16,686 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T05:55:16,707 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.099sec 2024-11-13T05:55:16,708 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T05:55:16,709 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T05:55:16,710 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T05:55:16,710 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T05:55:16,711 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T05:55:16,711 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,37807,1731477313603-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:55:16,712 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,37807,1731477313603-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T05:55:16,716 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T05:55:16,717 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T05:55:16,717 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,37807,1731477313603-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:55:16,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a900357, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:55:16,756 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T05:55:16,757 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T05:55:16,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,37807,-1 for getting cluster id 2024-11-13T05:55:16,763 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:55:16,772 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8e8c3ff9-23cc-4c45-b323-781f8a3738fe' 2024-11-13T05:55:16,774 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:55:16,774 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8e8c3ff9-23cc-4c45-b323-781f8a3738fe" 2024-11-13T05:55:16,775 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34831515, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:55:16,775 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,37807,-1] 2024-11-13T05:55:16,778 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:55:16,781 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:55:16,781 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48898, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:55:16,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36cdd3ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:55:16,784 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:55:16,791 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:55:16,791 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:55:16,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:55:16,831 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=bc386b7bf2fe,37807,1731477313603 2024-11-13T05:55:16,833 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-13T05:55:16,837 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:55:16,842 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is bc386b7bf2fe,37807,1731477313603 2024-11-13T05:55:16,844 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@12daeb33 2024-11-13T05:55:16,845 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:55:16,847 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:55:16,858 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:55:16,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37807 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T05:55:16,873 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T05:55:16,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true" procId is: 4 2024-11-13T05:55:16,875 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:16,878 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T05:55:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:55:16,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741837_1013 (size=1020) 2024-11-13T05:55:16,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741837_1013 (size=1020) 2024-11-13T05:55:16,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741837_1013 (size=1020) 2024-11-13T05:55:16,899 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 84316f2c5b758522338a907d7e8f386d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:16,900 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 200f36be3ca8364314512c33e4ae22d6, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:16,900 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => a840e8f158b2eaf5c08595bd38db426e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:16,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741838_1014 (size=107) 2024-11-13T05:55:16,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741838_1014 (size=107) 2024-11-13T05:55:16,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741838_1014 (size=107) 2024-11-13T05:55:16,920 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:16,920 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 84316f2c5b758522338a907d7e8f386d, disabling compactions & flushes 2024-11-13T05:55:16,920 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d. 2024-11-13T05:55:16,920 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d. 2024-11-13T05:55:16,920 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d. after waiting 0 ms 2024-11-13T05:55:16,920 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d. 2024-11-13T05:55:16,920 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d. 2024-11-13T05:55:16,920 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 84316f2c5b758522338a907d7e8f386d: Waiting for close lock at 1731477316920Disabling compacts and flushes for region at 1731477316920Disabling writes for close at 1731477316920Writing region close event to WAL at 1731477316920Closed at 1731477316920 2024-11-13T05:55:16,921 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e903b94793c361b461207b7aa3c133ac, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:16,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741840_1016 (size=110) 2024-11-13T05:55:16,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741840_1016 (size=110) 2024-11-13T05:55:16,927 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:16,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741840_1016 (size=110) 2024-11-13T05:55:16,927 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 200f36be3ca8364314512c33e4ae22d6, disabling compactions & flushes 2024-11-13T05:55:16,927 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6. 2024-11-13T05:55:16,927 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6. 2024-11-13T05:55:16,927 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6. after waiting 0 ms 2024-11-13T05:55:16,927 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6. 2024-11-13T05:55:16,927 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6. 2024-11-13T05:55:16,928 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 200f36be3ca8364314512c33e4ae22d6: Waiting for close lock at 1731477316927Disabling compacts and flushes for region at 1731477316927Disabling writes for close at 1731477316927Writing region close event to WAL at 1731477316927Closed at 1731477316927 2024-11-13T05:55:16,928 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 19321613b69704e42207868d450f785b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:16,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741839_1015 (size=110) 2024-11-13T05:55:16,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741839_1015 (size=110) 2024-11-13T05:55:16,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741839_1015 (size=110) 2024-11-13T05:55:16,939 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:16,939 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing a840e8f158b2eaf5c08595bd38db426e, disabling compactions & flushes 2024-11-13T05:55:16,939 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e. 2024-11-13T05:55:16,939 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e. 2024-11-13T05:55:16,940 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e. after waiting 0 ms 2024-11-13T05:55:16,940 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e. 2024-11-13T05:55:16,940 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e. 2024-11-13T05:55:16,940 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for a840e8f158b2eaf5c08595bd38db426e: Waiting for close lock at 1731477316939Disabling compacts and flushes for region at 1731477316939Disabling writes for close at 1731477316940 (+1 ms)Writing region close event to WAL at 1731477316940Closed at 1731477316940 2024-11-13T05:55:16,941 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1a492bcf61cc6ef0604137033fc7c90b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:16,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741841_1017 (size=110) 2024-11-13T05:55:16,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741841_1017 (size=110) 2024-11-13T05:55:16,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741841_1017 (size=110) 2024-11-13T05:55:16,951 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:16,951 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing e903b94793c361b461207b7aa3c133ac, disabling compactions & flushes 2024-11-13T05:55:16,951 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac. 2024-11-13T05:55:16,951 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac. 2024-11-13T05:55:16,951 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac. after waiting 0 ms 2024-11-13T05:55:16,951 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac. 2024-11-13T05:55:16,951 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac. 2024-11-13T05:55:16,951 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for e903b94793c361b461207b7aa3c133ac: Waiting for close lock at 1731477316951Disabling compacts and flushes for region at 1731477316951Disabling writes for close at 1731477316951Writing region close event to WAL at 1731477316951Closed at 1731477316951 2024-11-13T05:55:16,952 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 11abc6d6caf592b351052ae8adf50fd1, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:16,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741842_1018 (size=110) 2024-11-13T05:55:16,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741842_1018 (size=110) 2024-11-13T05:55:16,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741842_1018 (size=110) 2024-11-13T05:55:16,956 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:16,957 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 19321613b69704e42207868d450f785b, disabling compactions & flushes 2024-11-13T05:55:16,957 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b. 2024-11-13T05:55:16,957 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b. 2024-11-13T05:55:16,957 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b. after waiting 0 ms 2024-11-13T05:55:16,957 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b. 2024-11-13T05:55:16,957 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b. 2024-11-13T05:55:16,957 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 19321613b69704e42207868d450f785b: Waiting for close lock at 1731477316956Disabling compacts and flushes for region at 1731477316957 (+1 ms)Disabling writes for close at 1731477316957Writing region close event to WAL at 1731477316957Closed at 1731477316957 2024-11-13T05:55:16,958 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 408b59e776ffab17a3ee0e8fb465c67c, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:16,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741843_1019 (size=110) 2024-11-13T05:55:16,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741843_1019 (size=110) 2024-11-13T05:55:16,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741843_1019 (size=110) 2024-11-13T05:55:16,965 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:16,965 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 1a492bcf61cc6ef0604137033fc7c90b, disabling compactions & flushes 2024-11-13T05:55:16,965 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b. 2024-11-13T05:55:16,965 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b. 2024-11-13T05:55:16,965 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b. after waiting 0 ms 2024-11-13T05:55:16,965 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b. 2024-11-13T05:55:16,966 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b. 2024-11-13T05:55:16,966 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 1a492bcf61cc6ef0604137033fc7c90b: Waiting for close lock at 1731477316965Disabling compacts and flushes for region at 1731477316965Disabling writes for close at 1731477316965Writing region close event to WAL at 1731477316965Closed at 1731477316965 2024-11-13T05:55:16,967 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 89e8c9d6d5af67c75803894f4aea8ecc, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:16,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741844_1020 (size=110) 2024-11-13T05:55:16,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741844_1020 (size=110) 2024-11-13T05:55:16,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741844_1020 (size=110) 2024-11-13T05:55:16,971 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:16,971 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 11abc6d6caf592b351052ae8adf50fd1, disabling compactions & flushes 2024-11-13T05:55:16,971 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1. 2024-11-13T05:55:16,971 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1. 2024-11-13T05:55:16,971 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1. after waiting 0 ms 2024-11-13T05:55:16,971 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1. 2024-11-13T05:55:16,971 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1. 2024-11-13T05:55:16,972 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 11abc6d6caf592b351052ae8adf50fd1: Waiting for close lock at 1731477316971Disabling compacts and flushes for region at 1731477316971Disabling writes for close at 1731477316971Writing region close event to WAL at 1731477316971Closed at 1731477316971 2024-11-13T05:55:16,972 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4b613e7fdd93a34284c913bfa5a7b222, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:16,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741845_1021 (size=110) 2024-11-13T05:55:16,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741845_1021 (size=110) 2024-11-13T05:55:16,982 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:16,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741845_1021 (size=110) 2024-11-13T05:55:16,982 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 408b59e776ffab17a3ee0e8fb465c67c, disabling compactions & flushes 2024-11-13T05:55:16,982 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c. 2024-11-13T05:55:16,982 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c. 2024-11-13T05:55:16,982 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c. after waiting 0 ms 2024-11-13T05:55:16,983 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c. 2024-11-13T05:55:16,983 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c. 2024-11-13T05:55:16,983 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 408b59e776ffab17a3ee0e8fb465c67c: Waiting for close lock at 1731477316982Disabling compacts and flushes for region at 1731477316982Disabling writes for close at 1731477316983 (+1 ms)Writing region close event to WAL at 1731477316983Closed at 1731477316983 2024-11-13T05:55:16,984 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => b540ac2e8c4575b424d481848c4b4e52, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:16,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741846_1022 (size=110) 2024-11-13T05:55:16,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741846_1022 (size=110) 2024-11-13T05:55:16,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741846_1022 (size=110) 2024-11-13T05:55:16,986 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:16,986 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 89e8c9d6d5af67c75803894f4aea8ecc, disabling compactions & flushes 2024-11-13T05:55:16,986 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc. 2024-11-13T05:55:16,986 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc. 2024-11-13T05:55:16,986 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc. after waiting 0 ms 2024-11-13T05:55:16,986 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc. 2024-11-13T05:55:16,986 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc. 2024-11-13T05:55:16,986 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 89e8c9d6d5af67c75803894f4aea8ecc: Waiting for close lock at 1731477316986Disabling compacts and flushes for region at 1731477316986Disabling writes for close at 1731477316986Writing region close event to WAL at 1731477316986Closed at 1731477316986 2024-11-13T05:55:16,987 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 10c268d6da33b0a4590a478c0f32fd70, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:16,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:55:16,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741847_1023 (size=110) 2024-11-13T05:55:16,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741847_1023 (size=110) 2024-11-13T05:55:16,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741847_1023 (size=110) 2024-11-13T05:55:16,997 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:16,997 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 4b613e7fdd93a34284c913bfa5a7b222, disabling compactions & flushes 2024-11-13T05:55:16,997 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222. 2024-11-13T05:55:16,997 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222. 2024-11-13T05:55:16,997 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222. after waiting 0 ms 2024-11-13T05:55:16,997 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222. 2024-11-13T05:55:16,997 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222. 2024-11-13T05:55:16,997 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4b613e7fdd93a34284c913bfa5a7b222: Waiting for close lock at 1731477316997Disabling compacts and flushes for region at 1731477316997Disabling writes for close at 1731477316997Writing region close event to WAL at 1731477316997Closed at 1731477316997 2024-11-13T05:55:16,998 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5ff24ce91fac66115879ac4ec57a346a, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:17,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741848_1024 (size=110) 2024-11-13T05:55:17,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741848_1024 (size=110) 2024-11-13T05:55:17,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741848_1024 (size=110) 2024-11-13T05:55:17,006 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,007 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing b540ac2e8c4575b424d481848c4b4e52, disabling compactions & flushes 2024-11-13T05:55:17,007 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52. 2024-11-13T05:55:17,007 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52. 2024-11-13T05:55:17,007 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52. after waiting 0 ms 2024-11-13T05:55:17,007 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52. 2024-11-13T05:55:17,007 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52. 2024-11-13T05:55:17,007 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for b540ac2e8c4575b424d481848c4b4e52: Waiting for close lock at 1731477317007Disabling compacts and flushes for region at 1731477317007Disabling writes for close at 1731477317007Writing region close event to WAL at 1731477317007Closed at 1731477317007 2024-11-13T05:55:17,008 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => f7c7452ce5410ec8dcf456adb9b7db6b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:17,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741849_1025 (size=110) 2024-11-13T05:55:17,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741849_1025 (size=110) 2024-11-13T05:55:17,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741849_1025 (size=110) 2024-11-13T05:55:17,016 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,017 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 10c268d6da33b0a4590a478c0f32fd70, disabling compactions & flushes 2024-11-13T05:55:17,017 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70. 2024-11-13T05:55:17,017 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70. 2024-11-13T05:55:17,017 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70. after waiting 0 ms 2024-11-13T05:55:17,017 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70. 2024-11-13T05:55:17,017 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70. 2024-11-13T05:55:17,017 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 10c268d6da33b0a4590a478c0f32fd70: Waiting for close lock at 1731477317016Disabling compacts and flushes for region at 1731477317016Disabling writes for close at 1731477317017 (+1 ms)Writing region close event to WAL at 1731477317017Closed at 1731477317017 2024-11-13T05:55:17,018 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 43fd53cccd0bd76519c89a78d6bf6131, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:17,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741850_1026 (size=110) 2024-11-13T05:55:17,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741850_1026 (size=110) 2024-11-13T05:55:17,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741850_1026 (size=110) 2024-11-13T05:55:17,024 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,024 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 5ff24ce91fac66115879ac4ec57a346a, disabling compactions & flushes 2024-11-13T05:55:17,024 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a. 2024-11-13T05:55:17,024 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a. 2024-11-13T05:55:17,024 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a. after waiting 0 ms 2024-11-13T05:55:17,024 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a. 2024-11-13T05:55:17,024 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a. 2024-11-13T05:55:17,024 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5ff24ce91fac66115879ac4ec57a346a: Waiting for close lock at 1731477317024Disabling compacts and flushes for region at 1731477317024Disabling writes for close at 1731477317024Writing region close event to WAL at 1731477317024Closed at 1731477317024 2024-11-13T05:55:17,025 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9afb90ba9af49bb3547d4438986e1fca, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:17,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741851_1027 (size=110) 2024-11-13T05:55:17,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741851_1027 (size=110) 2024-11-13T05:55:17,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741851_1027 (size=110) 2024-11-13T05:55:17,032 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,032 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing f7c7452ce5410ec8dcf456adb9b7db6b, disabling compactions & flushes 2024-11-13T05:55:17,032 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b. 2024-11-13T05:55:17,032 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b. 2024-11-13T05:55:17,032 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b. after waiting 0 ms 2024-11-13T05:55:17,032 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b. 2024-11-13T05:55:17,032 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b. 2024-11-13T05:55:17,032 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for f7c7452ce5410ec8dcf456adb9b7db6b: Waiting for close lock at 1731477317032Disabling compacts and flushes for region at 1731477317032Disabling writes for close at 1731477317032Writing region close event to WAL at 1731477317032Closed at 1731477317032 2024-11-13T05:55:17,033 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 408dfc042fd5f2d964a15b01fd44966c, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:17,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741852_1028 (size=110) 2024-11-13T05:55:17,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741852_1028 (size=110) 2024-11-13T05:55:17,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741852_1028 (size=110) 2024-11-13T05:55:17,041 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,042 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 43fd53cccd0bd76519c89a78d6bf6131, disabling compactions & flushes 2024-11-13T05:55:17,042 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131. 2024-11-13T05:55:17,042 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131. 2024-11-13T05:55:17,042 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131. after waiting 0 ms 2024-11-13T05:55:17,042 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131. 2024-11-13T05:55:17,042 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131. 2024-11-13T05:55:17,042 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 43fd53cccd0bd76519c89a78d6bf6131: Waiting for close lock at 1731477317042Disabling compacts and flushes for region at 1731477317042Disabling writes for close at 1731477317042Writing region close event to WAL at 1731477317042Closed at 1731477317042 2024-11-13T05:55:17,043 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 9bd21adcd80ce6ee8c5ed1a581c04749, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:17,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741853_1029 (size=110) 2024-11-13T05:55:17,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741853_1029 (size=110) 2024-11-13T05:55:17,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741853_1029 (size=110) 2024-11-13T05:55:17,048 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,048 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 9afb90ba9af49bb3547d4438986e1fca, disabling compactions & flushes 2024-11-13T05:55:17,048 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca. 2024-11-13T05:55:17,048 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca. 2024-11-13T05:55:17,048 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca. after waiting 0 ms 2024-11-13T05:55:17,049 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca. 2024-11-13T05:55:17,049 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca. 2024-11-13T05:55:17,049 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9afb90ba9af49bb3547d4438986e1fca: Waiting for close lock at 1731477317048Disabling compacts and flushes for region at 1731477317048Disabling writes for close at 1731477317049 (+1 ms)Writing region close event to WAL at 1731477317049Closed at 1731477317049 2024-11-13T05:55:17,050 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 82c0a84200b3abe2cb346258143147c7, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:17,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741854_1030 (size=110) 2024-11-13T05:55:17,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741854_1030 (size=110) 2024-11-13T05:55:17,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741854_1030 (size=110) 2024-11-13T05:55:17,057 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,057 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 408dfc042fd5f2d964a15b01fd44966c, disabling compactions & flushes 2024-11-13T05:55:17,057 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c. 2024-11-13T05:55:17,057 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c. 2024-11-13T05:55:17,057 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c. after waiting 0 ms 2024-11-13T05:55:17,057 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c. 2024-11-13T05:55:17,057 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c. 2024-11-13T05:55:17,057 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 408dfc042fd5f2d964a15b01fd44966c: Waiting for close lock at 1731477317057Disabling compacts and flushes for region at 1731477317057Disabling writes for close at 1731477317057Writing region close event to WAL at 1731477317057Closed at 1731477317057 2024-11-13T05:55:17,058 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 2810327548325125e764254e87017046, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:17,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741855_1031 (size=110) 2024-11-13T05:55:17,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741855_1031 (size=110) 2024-11-13T05:55:17,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741855_1031 (size=110) 2024-11-13T05:55:17,063 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,063 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 9bd21adcd80ce6ee8c5ed1a581c04749, disabling compactions & flushes 2024-11-13T05:55:17,063 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749. 2024-11-13T05:55:17,063 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749. 2024-11-13T05:55:17,063 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749. after waiting 0 ms 2024-11-13T05:55:17,063 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749. 2024-11-13T05:55:17,063 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749. 2024-11-13T05:55:17,064 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 9bd21adcd80ce6ee8c5ed1a581c04749: Waiting for close lock at 1731477317063Disabling compacts and flushes for region at 1731477317063Disabling writes for close at 1731477317063Writing region close event to WAL at 1731477317063Closed at 1731477317063 2024-11-13T05:55:17,064 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => e86ecaa5f83edede0f153db264078a94, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:17,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741856_1032 (size=110) 2024-11-13T05:55:17,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741856_1032 (size=110) 2024-11-13T05:55:17,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741856_1032 (size=110) 2024-11-13T05:55:17,072 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,072 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 82c0a84200b3abe2cb346258143147c7, disabling compactions & flushes 2024-11-13T05:55:17,072 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7. 2024-11-13T05:55:17,072 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7. 2024-11-13T05:55:17,072 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7. after waiting 0 ms 2024-11-13T05:55:17,072 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7. 2024-11-13T05:55:17,072 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7. 2024-11-13T05:55:17,072 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 82c0a84200b3abe2cb346258143147c7: Waiting for close lock at 1731477317072Disabling compacts and flushes for region at 1731477317072Disabling writes for close at 1731477317072Writing region close event to WAL at 1731477317072Closed at 1731477317072 2024-11-13T05:55:17,073 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a65241dbe6c2863f31a0138a73130225, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:17,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741858_1034 (size=110) 2024-11-13T05:55:17,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741857_1033 (size=110) 2024-11-13T05:55:17,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741857_1033 (size=110) 2024-11-13T05:55:17,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741858_1034 (size=110) 2024-11-13T05:55:17,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741857_1033 (size=110) 2024-11-13T05:55:17,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741858_1034 (size=110) 2024-11-13T05:55:17,078 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,078 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 2810327548325125e764254e87017046, disabling compactions & flushes 2024-11-13T05:55:17,078 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046. 2024-11-13T05:55:17,079 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046. 2024-11-13T05:55:17,079 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046. after waiting 0 ms 2024-11-13T05:55:17,079 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046. 2024-11-13T05:55:17,079 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046. 2024-11-13T05:55:17,079 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 2810327548325125e764254e87017046: Waiting for close lock at 1731477317078Disabling compacts and flushes for region at 1731477317078Disabling writes for close at 1731477317079 (+1 ms)Writing region close event to WAL at 1731477317079Closed at 1731477317079 2024-11-13T05:55:17,079 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,080 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ff62c0b5be01546e2b1563e065750a3b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:17,080 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing e86ecaa5f83edede0f153db264078a94, disabling compactions & flushes 2024-11-13T05:55:17,080 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94. 2024-11-13T05:55:17,080 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94. 2024-11-13T05:55:17,080 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94. after waiting 0 ms 2024-11-13T05:55:17,080 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94. 2024-11-13T05:55:17,080 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94. 2024-11-13T05:55:17,080 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for e86ecaa5f83edede0f153db264078a94: Waiting for close lock at 1731477317080Disabling compacts and flushes for region at 1731477317080Disabling writes for close at 1731477317080Writing region close event to WAL at 1731477317080Closed at 1731477317080 2024-11-13T05:55:17,081 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 80710108d60e6f4a1b8975ccfe51e50e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:17,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741859_1035 (size=110) 2024-11-13T05:55:17,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741859_1035 (size=110) 2024-11-13T05:55:17,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741859_1035 (size=110) 2024-11-13T05:55:17,092 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,092 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing a65241dbe6c2863f31a0138a73130225, disabling compactions & flushes 2024-11-13T05:55:17,092 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225. 2024-11-13T05:55:17,092 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225. 2024-11-13T05:55:17,092 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225. after waiting 0 ms 2024-11-13T05:55:17,093 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225. 2024-11-13T05:55:17,093 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225. 2024-11-13T05:55:17,093 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for a65241dbe6c2863f31a0138a73130225: Waiting for close lock at 1731477317092Disabling compacts and flushes for region at 1731477317092Disabling writes for close at 1731477317093 (+1 ms)Writing region close event to WAL at 1731477317093Closed at 1731477317093 2024-11-13T05:55:17,094 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 98cb12bd0a37bbbfee08e3459a85f735, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:17,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741860_1036 (size=110) 2024-11-13T05:55:17,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741860_1036 (size=110) 2024-11-13T05:55:17,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741860_1036 (size=110) 2024-11-13T05:55:17,101 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,101 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing ff62c0b5be01546e2b1563e065750a3b, disabling compactions & flushes 2024-11-13T05:55:17,101 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b. 2024-11-13T05:55:17,102 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b. 2024-11-13T05:55:17,102 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b. after waiting 0 ms 2024-11-13T05:55:17,102 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b. 2024-11-13T05:55:17,102 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b. 2024-11-13T05:55:17,102 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for ff62c0b5be01546e2b1563e065750a3b: Waiting for close lock at 1731477317101Disabling compacts and flushes for region at 1731477317101Disabling writes for close at 1731477317102 (+1 ms)Writing region close event to WAL at 1731477317102Closed at 1731477317102 2024-11-13T05:55:17,103 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 90b03edd8043eabbad2ddbd32409c478, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455 2024-11-13T05:55:17,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741861_1037 (size=110) 2024-11-13T05:55:17,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741861_1037 (size=110) 2024-11-13T05:55:17,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741861_1037 (size=110) 2024-11-13T05:55:17,111 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741862_1038 (size=110) 2024-11-13T05:55:17,112 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 80710108d60e6f4a1b8975ccfe51e50e, disabling compactions & flushes 2024-11-13T05:55:17,112 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e. 2024-11-13T05:55:17,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741862_1038 (size=110) 2024-11-13T05:55:17,112 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e. 2024-11-13T05:55:17,112 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e. after waiting 0 ms 2024-11-13T05:55:17,112 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e. 2024-11-13T05:55:17,112 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e. 2024-11-13T05:55:17,112 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 80710108d60e6f4a1b8975ccfe51e50e: Waiting for close lock at 1731477317111Disabling compacts and flushes for region at 1731477317111Disabling writes for close at 1731477317112 (+1 ms)Writing region close event to WAL at 1731477317112Closed at 1731477317112 2024-11-13T05:55:17,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741862_1038 (size=110) 2024-11-13T05:55:17,114 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,115 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 98cb12bd0a37bbbfee08e3459a85f735, disabling compactions & flushes 2024-11-13T05:55:17,115 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735. 2024-11-13T05:55:17,115 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735. 2024-11-13T05:55:17,115 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735. after waiting 0 ms 2024-11-13T05:55:17,115 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735. 2024-11-13T05:55:17,115 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735. 2024-11-13T05:55:17,115 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 98cb12bd0a37bbbfee08e3459a85f735: Waiting for close lock at 1731477317115Disabling compacts and flushes for region at 1731477317115Disabling writes for close at 1731477317115Writing region close event to WAL at 1731477317115Closed at 1731477317115 2024-11-13T05:55:17,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741863_1039 (size=107) 2024-11-13T05:55:17,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741863_1039 (size=107) 2024-11-13T05:55:17,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741863_1039 (size=107) 2024-11-13T05:55:17,120 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,120 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 90b03edd8043eabbad2ddbd32409c478, disabling compactions & flushes 2024-11-13T05:55:17,120 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478. 2024-11-13T05:55:17,120 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478. 2024-11-13T05:55:17,120 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478. after waiting 0 ms 2024-11-13T05:55:17,120 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478. 2024-11-13T05:55:17,120 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478. 2024-11-13T05:55:17,121 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 90b03edd8043eabbad2ddbd32409c478: Waiting for close lock at 1731477317120Disabling compacts and flushes for region at 1731477317120Disabling writes for close at 1731477317120Writing region close event to WAL at 1731477317120Closed at 1731477317120 2024-11-13T05:55:17,124 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T05:55:17,129 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d.","families":{"info":[{"qualifier":"regioninfo","vlen":106,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,129 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,130 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,130 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,130 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,130 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,131 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,131 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,131 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,131 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,132 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,132 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,132 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,132 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,132 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,133 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,133 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,133 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,133 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,134 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,134 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,134 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,134 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,134 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,135 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,135 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478.","families":{"info":[{"qualifier":"regioninfo","vlen":106,"tag":[],"timestamp":"1731477317124"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477317124"}]},"ts":"1731477317124"} 2024-11-13T05:55:17,182 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-13T05:55:17,184 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T05:55:17,187 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477317184"}]},"ts":"1731477317184"} 2024-11-13T05:55:17,191 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, state=ENABLING in hbase:meta 2024-11-13T05:55:17,192 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {bc386b7bf2fe=0} racks are {/default-rack=0} 2024-11-13T05:55:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:55:17,230 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T05:55:17,230 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T05:55:17,231 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T05:55:17,231 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T05:55:17,231 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T05:55:17,231 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T05:55:17,231 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T05:55:17,231 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T05:55:17,231 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T05:55:17,231 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T05:55:17,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=84316f2c5b758522338a907d7e8f386d, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=200f36be3ca8364314512c33e4ae22d6, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a840e8f158b2eaf5c08595bd38db426e, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e903b94793c361b461207b7aa3c133ac, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=19321613b69704e42207868d450f785b, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1a492bcf61cc6ef0604137033fc7c90b, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=11abc6d6caf592b351052ae8adf50fd1, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=408b59e776ffab17a3ee0e8fb465c67c, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=89e8c9d6d5af67c75803894f4aea8ecc, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4b613e7fdd93a34284c913bfa5a7b222, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b540ac2e8c4575b424d481848c4b4e52, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=10c268d6da33b0a4590a478c0f32fd70, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ff24ce91fac66115879ac4ec57a346a, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f7c7452ce5410ec8dcf456adb9b7db6b, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=43fd53cccd0bd76519c89a78d6bf6131, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9afb90ba9af49bb3547d4438986e1fca, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=408dfc042fd5f2d964a15b01fd44966c, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9bd21adcd80ce6ee8c5ed1a581c04749, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=82c0a84200b3abe2cb346258143147c7, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2810327548325125e764254e87017046, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e86ecaa5f83edede0f153db264078a94, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a65241dbe6c2863f31a0138a73130225, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ff62c0b5be01546e2b1563e065750a3b, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=80710108d60e6f4a1b8975ccfe51e50e, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=98cb12bd0a37bbbfee08e3459a85f735, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=90b03edd8043eabbad2ddbd32409c478, ASSIGN}] 2024-11-13T05:55:17,243 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=90b03edd8043eabbad2ddbd32409c478, ASSIGN 2024-11-13T05:55:17,245 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=90b03edd8043eabbad2ddbd32409c478, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,45383,1731477314448; forceNewPlan=false, retain=false 2024-11-13T05:55:17,247 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=98cb12bd0a37bbbfee08e3459a85f735, ASSIGN 2024-11-13T05:55:17,248 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=98cb12bd0a37bbbfee08e3459a85f735, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,38823,1731477314332; forceNewPlan=false, retain=false 2024-11-13T05:55:17,250 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=80710108d60e6f4a1b8975ccfe51e50e, ASSIGN 2024-11-13T05:55:17,252 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=80710108d60e6f4a1b8975ccfe51e50e, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,40327,1731477314499; forceNewPlan=false, retain=false 2024-11-13T05:55:17,253 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ff62c0b5be01546e2b1563e065750a3b, ASSIGN 2024-11-13T05:55:17,254 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ff62c0b5be01546e2b1563e065750a3b, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,45383,1731477314448; forceNewPlan=false, retain=false 2024-11-13T05:55:17,256 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a65241dbe6c2863f31a0138a73130225, ASSIGN 2024-11-13T05:55:17,257 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a65241dbe6c2863f31a0138a73130225, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,38823,1731477314332; forceNewPlan=false, retain=false 2024-11-13T05:55:17,259 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e86ecaa5f83edede0f153db264078a94, ASSIGN 2024-11-13T05:55:17,261 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e86ecaa5f83edede0f153db264078a94, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,40327,1731477314499; forceNewPlan=false, retain=false 2024-11-13T05:55:17,262 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2810327548325125e764254e87017046, ASSIGN 2024-11-13T05:55:17,264 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2810327548325125e764254e87017046, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,45383,1731477314448; forceNewPlan=false, retain=false 2024-11-13T05:55:17,265 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=82c0a84200b3abe2cb346258143147c7, ASSIGN 2024-11-13T05:55:17,267 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=82c0a84200b3abe2cb346258143147c7, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,38823,1731477314332; forceNewPlan=false, retain=false 2024-11-13T05:55:17,268 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9bd21adcd80ce6ee8c5ed1a581c04749, ASSIGN 2024-11-13T05:55:17,270 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9bd21adcd80ce6ee8c5ed1a581c04749, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,40327,1731477314499; forceNewPlan=false, retain=false 2024-11-13T05:55:17,271 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=408dfc042fd5f2d964a15b01fd44966c, ASSIGN 2024-11-13T05:55:17,273 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=408dfc042fd5f2d964a15b01fd44966c, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,45383,1731477314448; forceNewPlan=false, retain=false 2024-11-13T05:55:17,274 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9afb90ba9af49bb3547d4438986e1fca, ASSIGN 2024-11-13T05:55:17,275 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9afb90ba9af49bb3547d4438986e1fca, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,38823,1731477314332; forceNewPlan=false, retain=false 2024-11-13T05:55:17,277 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=43fd53cccd0bd76519c89a78d6bf6131, ASSIGN 2024-11-13T05:55:17,278 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=43fd53cccd0bd76519c89a78d6bf6131, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,40327,1731477314499; forceNewPlan=false, retain=false 2024-11-13T05:55:17,279 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f7c7452ce5410ec8dcf456adb9b7db6b, ASSIGN 2024-11-13T05:55:17,281 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f7c7452ce5410ec8dcf456adb9b7db6b, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,45383,1731477314448; forceNewPlan=false, retain=false 2024-11-13T05:55:17,283 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ff24ce91fac66115879ac4ec57a346a, ASSIGN 2024-11-13T05:55:17,284 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ff24ce91fac66115879ac4ec57a346a, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,38823,1731477314332; forceNewPlan=false, retain=false 2024-11-13T05:55:17,286 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=10c268d6da33b0a4590a478c0f32fd70, ASSIGN 2024-11-13T05:55:17,287 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=10c268d6da33b0a4590a478c0f32fd70, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,40327,1731477314499; forceNewPlan=false, retain=false 2024-11-13T05:55:17,289 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b540ac2e8c4575b424d481848c4b4e52, ASSIGN 2024-11-13T05:55:17,291 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b540ac2e8c4575b424d481848c4b4e52, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,45383,1731477314448; forceNewPlan=false, retain=false 2024-11-13T05:55:17,293 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4b613e7fdd93a34284c913bfa5a7b222, ASSIGN 2024-11-13T05:55:17,295 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4b613e7fdd93a34284c913bfa5a7b222, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,38823,1731477314332; forceNewPlan=false, retain=false 2024-11-13T05:55:17,296 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=89e8c9d6d5af67c75803894f4aea8ecc, ASSIGN 2024-11-13T05:55:17,298 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=89e8c9d6d5af67c75803894f4aea8ecc, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,40327,1731477314499; forceNewPlan=false, retain=false 2024-11-13T05:55:17,300 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=408b59e776ffab17a3ee0e8fb465c67c, ASSIGN 2024-11-13T05:55:17,301 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=408b59e776ffab17a3ee0e8fb465c67c, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,45383,1731477314448; forceNewPlan=false, retain=false 2024-11-13T05:55:17,303 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=11abc6d6caf592b351052ae8adf50fd1, ASSIGN 2024-11-13T05:55:17,305 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=11abc6d6caf592b351052ae8adf50fd1, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,38823,1731477314332; forceNewPlan=false, retain=false 2024-11-13T05:55:17,307 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1a492bcf61cc6ef0604137033fc7c90b, ASSIGN 2024-11-13T05:55:17,308 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1a492bcf61cc6ef0604137033fc7c90b, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,40327,1731477314499; forceNewPlan=false, retain=false 2024-11-13T05:55:17,310 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=19321613b69704e42207868d450f785b, ASSIGN 2024-11-13T05:55:17,311 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=19321613b69704e42207868d450f785b, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,45383,1731477314448; forceNewPlan=false, retain=false 2024-11-13T05:55:17,313 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e903b94793c361b461207b7aa3c133ac, ASSIGN 2024-11-13T05:55:17,314 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e903b94793c361b461207b7aa3c133ac, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,38823,1731477314332; forceNewPlan=false, retain=false 2024-11-13T05:55:17,316 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a840e8f158b2eaf5c08595bd38db426e, ASSIGN 2024-11-13T05:55:17,318 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a840e8f158b2eaf5c08595bd38db426e, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,40327,1731477314499; forceNewPlan=false, retain=false 2024-11-13T05:55:17,320 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=200f36be3ca8364314512c33e4ae22d6, ASSIGN 2024-11-13T05:55:17,321 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=200f36be3ca8364314512c33e4ae22d6, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,45383,1731477314448; forceNewPlan=false, retain=false 2024-11-13T05:55:17,323 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=84316f2c5b758522338a907d7e8f386d, ASSIGN 2024-11-13T05:55:17,324 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=84316f2c5b758522338a907d7e8f386d, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,38823,1731477314332; forceNewPlan=false, retain=false 2024-11-13T05:55:17,399 INFO [bc386b7bf2fe:37807 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-13T05:55:17,400 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=90b03edd8043eabbad2ddbd32409c478, regionState=OPENING, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:17,401 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=ff62c0b5be01546e2b1563e065750a3b, regionState=OPENING, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:17,402 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=2810327548325125e764254e87017046, regionState=OPENING, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:17,403 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=408dfc042fd5f2d964a15b01fd44966c, regionState=OPENING, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:17,404 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=f7c7452ce5410ec8dcf456adb9b7db6b, regionState=OPENING, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:17,404 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=b540ac2e8c4575b424d481848c4b4e52, regionState=OPENING, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:17,405 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=90b03edd8043eabbad2ddbd32409c478, ASSIGN because future has completed 2024-11-13T05:55:17,405 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=408b59e776ffab17a3ee0e8fb465c67c, regionState=OPENING, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:17,406 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=19321613b69704e42207868d450f785b, regionState=OPENING, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:17,407 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=200f36be3ca8364314512c33e4ae22d6, regionState=OPENING, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:17,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90b03edd8043eabbad2ddbd32409c478, server=bc386b7bf2fe,45383,1731477314448}] 2024-11-13T05:55:17,408 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ff62c0b5be01546e2b1563e065750a3b, ASSIGN because future has completed 2024-11-13T05:55:17,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2810327548325125e764254e87017046, ASSIGN because future has completed 2024-11-13T05:55:17,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=408dfc042fd5f2d964a15b01fd44966c, ASSIGN because future has completed 2024-11-13T05:55:17,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 408dfc042fd5f2d964a15b01fd44966c, server=bc386b7bf2fe,45383,1731477314448}] 2024-11-13T05:55:17,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f7c7452ce5410ec8dcf456adb9b7db6b, ASSIGN because future has completed 2024-11-13T05:55:17,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b540ac2e8c4575b424d481848c4b4e52, ASSIGN because future has completed 2024-11-13T05:55:17,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=408b59e776ffab17a3ee0e8fb465c67c, ASSIGN because future has completed 2024-11-13T05:55:17,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=19321613b69704e42207868d450f785b, ASSIGN because future has completed 2024-11-13T05:55:17,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 19321613b69704e42207868d450f785b, server=bc386b7bf2fe,45383,1731477314448}] 2024-11-13T05:55:17,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=200f36be3ca8364314512c33e4ae22d6, ASSIGN because future has completed 2024-11-13T05:55:17,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 200f36be3ca8364314512c33e4ae22d6, server=bc386b7bf2fe,45383,1731477314448}] 2024-11-13T05:55:17,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 408b59e776ffab17a3ee0e8fb465c67c, server=bc386b7bf2fe,45383,1731477314448}] 2024-11-13T05:55:17,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure b540ac2e8c4575b424d481848c4b4e52, server=bc386b7bf2fe,45383,1731477314448}] 2024-11-13T05:55:17,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7c7452ce5410ec8dcf456adb9b7db6b, server=bc386b7bf2fe,45383,1731477314448}] 2024-11-13T05:55:17,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2810327548325125e764254e87017046, server=bc386b7bf2fe,45383,1731477314448}] 2024-11-13T05:55:17,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff62c0b5be01546e2b1563e065750a3b, server=bc386b7bf2fe,45383,1731477314448}] 2024-11-13T05:55:17,469 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=80710108d60e6f4a1b8975ccfe51e50e, regionState=OPENING, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:55:17,469 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=e86ecaa5f83edede0f153db264078a94, regionState=OPENING, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:55:17,470 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=9bd21adcd80ce6ee8c5ed1a581c04749, regionState=OPENING, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:55:17,471 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=43fd53cccd0bd76519c89a78d6bf6131, regionState=OPENING, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:55:17,471 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=10c268d6da33b0a4590a478c0f32fd70, regionState=OPENING, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:55:17,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=80710108d60e6f4a1b8975ccfe51e50e, ASSIGN because future has completed 2024-11-13T05:55:17,472 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=89e8c9d6d5af67c75803894f4aea8ecc, regionState=OPENING, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:55:17,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80710108d60e6f4a1b8975ccfe51e50e, server=bc386b7bf2fe,40327,1731477314499}] 2024-11-13T05:55:17,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e86ecaa5f83edede0f153db264078a94, ASSIGN because future has completed 2024-11-13T05:55:17,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure e86ecaa5f83edede0f153db264078a94, server=bc386b7bf2fe,40327,1731477314499}] 2024-11-13T05:55:17,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9bd21adcd80ce6ee8c5ed1a581c04749, ASSIGN because future has completed 2024-11-13T05:55:17,478 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=43fd53cccd0bd76519c89a78d6bf6131, ASSIGN because future has completed 2024-11-13T05:55:17,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=10c268d6da33b0a4590a478c0f32fd70, ASSIGN because future has completed 2024-11-13T05:55:17,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10c268d6da33b0a4590a478c0f32fd70, server=bc386b7bf2fe,40327,1731477314499}] 2024-11-13T05:55:17,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=89e8c9d6d5af67c75803894f4aea8ecc, ASSIGN because future has completed 2024-11-13T05:55:17,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89e8c9d6d5af67c75803894f4aea8ecc, server=bc386b7bf2fe,40327,1731477314499}] 2024-11-13T05:55:17,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43fd53cccd0bd76519c89a78d6bf6131, server=bc386b7bf2fe,40327,1731477314499}] 2024-11-13T05:55:17,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9bd21adcd80ce6ee8c5ed1a581c04749, server=bc386b7bf2fe,40327,1731477314499}] 2024-11-13T05:55:17,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:55:17,510 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1a492bcf61cc6ef0604137033fc7c90b, regionState=OPENING, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:55:17,510 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a840e8f158b2eaf5c08595bd38db426e, regionState=OPENING, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:55:17,511 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=98cb12bd0a37bbbfee08e3459a85f735, regionState=OPENING, regionLocation=bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:17,512 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=a65241dbe6c2863f31a0138a73130225, regionState=OPENING, regionLocation=bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:17,513 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=82c0a84200b3abe2cb346258143147c7, regionState=OPENING, regionLocation=bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:17,514 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=9afb90ba9af49bb3547d4438986e1fca, regionState=OPENING, regionLocation=bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:17,514 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=5ff24ce91fac66115879ac4ec57a346a, regionState=OPENING, regionLocation=bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:17,515 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=4b613e7fdd93a34284c913bfa5a7b222, regionState=OPENING, regionLocation=bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:17,516 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=11abc6d6caf592b351052ae8adf50fd1, regionState=OPENING, regionLocation=bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:17,516 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=e903b94793c361b461207b7aa3c133ac, regionState=OPENING, regionLocation=bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:17,517 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=84316f2c5b758522338a907d7e8f386d, regionState=OPENING, regionLocation=bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:17,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1a492bcf61cc6ef0604137033fc7c90b, ASSIGN because future has completed 2024-11-13T05:55:17,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a492bcf61cc6ef0604137033fc7c90b, server=bc386b7bf2fe,40327,1731477314499}] 2024-11-13T05:55:17,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a840e8f158b2eaf5c08595bd38db426e, ASSIGN because future has completed 2024-11-13T05:55:17,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=98cb12bd0a37bbbfee08e3459a85f735, ASSIGN because future has completed 2024-11-13T05:55:17,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a65241dbe6c2863f31a0138a73130225, ASSIGN because future has completed 2024-11-13T05:55:17,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=82c0a84200b3abe2cb346258143147c7, ASSIGN because future has completed 2024-11-13T05:55:17,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82c0a84200b3abe2cb346258143147c7, server=bc386b7bf2fe,38823,1731477314332}] 2024-11-13T05:55:17,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9afb90ba9af49bb3547d4438986e1fca, ASSIGN because future has completed 2024-11-13T05:55:17,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ff24ce91fac66115879ac4ec57a346a, ASSIGN because future has completed 2024-11-13T05:55:17,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4b613e7fdd93a34284c913bfa5a7b222, ASSIGN because future has completed 2024-11-13T05:55:17,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b613e7fdd93a34284c913bfa5a7b222, server=bc386b7bf2fe,38823,1731477314332}] 2024-11-13T05:55:17,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=11abc6d6caf592b351052ae8adf50fd1, ASSIGN because future has completed 2024-11-13T05:55:17,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e903b94793c361b461207b7aa3c133ac, ASSIGN because future has completed 2024-11-13T05:55:17,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=84316f2c5b758522338a907d7e8f386d, ASSIGN because future has completed 2024-11-13T05:55:17,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 84316f2c5b758522338a907d7e8f386d, server=bc386b7bf2fe,38823,1731477314332}] 2024-11-13T05:55:17,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure e903b94793c361b461207b7aa3c133ac, server=bc386b7bf2fe,38823,1731477314332}] 2024-11-13T05:55:17,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11abc6d6caf592b351052ae8adf50fd1, server=bc386b7bf2fe,38823,1731477314332}] 2024-11-13T05:55:17,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ff24ce91fac66115879ac4ec57a346a, server=bc386b7bf2fe,38823,1731477314332}] 2024-11-13T05:55:17,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9afb90ba9af49bb3547d4438986e1fca, server=bc386b7bf2fe,38823,1731477314332}] 2024-11-13T05:55:17,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure a65241dbe6c2863f31a0138a73130225, server=bc386b7bf2fe,38823,1731477314332}] 2024-11-13T05:55:17,565 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T05:55:17,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 98cb12bd0a37bbbfee08e3459a85f735, server=bc386b7bf2fe,38823,1731477314332}] 2024-11-13T05:55:17,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure a840e8f158b2eaf5c08595bd38db426e, server=bc386b7bf2fe,40327,1731477314499}] 2024-11-13T05:55:17,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50487, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:55:17,578 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6. 2024-11-13T05:55:17,578 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 200f36be3ca8364314512c33e4ae22d6, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-13T05:55:17,578 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 200f36be3ca8364314512c33e4ae22d6 2024-11-13T05:55:17,579 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,579 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 200f36be3ca8364314512c33e4ae22d6 2024-11-13T05:55:17,579 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 200f36be3ca8364314512c33e4ae22d6 2024-11-13T05:55:17,581 INFO [StoreOpener-200f36be3ca8364314512c33e4ae22d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 200f36be3ca8364314512c33e4ae22d6 2024-11-13T05:55:17,583 INFO [StoreOpener-200f36be3ca8364314512c33e4ae22d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 200f36be3ca8364314512c33e4ae22d6 columnFamilyName colfamily11 2024-11-13T05:55:17,583 DEBUG [StoreOpener-200f36be3ca8364314512c33e4ae22d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,584 INFO [StoreOpener-200f36be3ca8364314512c33e4ae22d6-1 {}] regionserver.HStore(327): Store=200f36be3ca8364314512c33e4ae22d6/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,585 INFO [StoreOpener-200f36be3ca8364314512c33e4ae22d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 200f36be3ca8364314512c33e4ae22d6 2024-11-13T05:55:17,586 INFO [StoreOpener-200f36be3ca8364314512c33e4ae22d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 200f36be3ca8364314512c33e4ae22d6 columnFamilyName colfamily21 2024-11-13T05:55:17,587 DEBUG [StoreOpener-200f36be3ca8364314512c33e4ae22d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,587 INFO [StoreOpener-200f36be3ca8364314512c33e4ae22d6-1 {}] regionserver.HStore(327): Store=200f36be3ca8364314512c33e4ae22d6/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,587 INFO [StoreOpener-200f36be3ca8364314512c33e4ae22d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 200f36be3ca8364314512c33e4ae22d6 2024-11-13T05:55:17,589 INFO [StoreOpener-200f36be3ca8364314512c33e4ae22d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 200f36be3ca8364314512c33e4ae22d6 columnFamilyName colfamily31 2024-11-13T05:55:17,590 DEBUG [StoreOpener-200f36be3ca8364314512c33e4ae22d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,590 INFO [StoreOpener-200f36be3ca8364314512c33e4ae22d6-1 {}] regionserver.HStore(327): Store=200f36be3ca8364314512c33e4ae22d6/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,591 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 200f36be3ca8364314512c33e4ae22d6 2024-11-13T05:55:17,593 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/200f36be3ca8364314512c33e4ae22d6 2024-11-13T05:55:17,594 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/200f36be3ca8364314512c33e4ae22d6 2024-11-13T05:55:17,595 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 200f36be3ca8364314512c33e4ae22d6 2024-11-13T05:55:17,595 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 200f36be3ca8364314512c33e4ae22d6 2024-11-13T05:55:17,596 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,599 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 200f36be3ca8364314512c33e4ae22d6 2024-11-13T05:55:17,602 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/200f36be3ca8364314512c33e4ae22d6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,602 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 200f36be3ca8364314512c33e4ae22d6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68234356, jitterRate=0.016771137714385986}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,603 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 200f36be3ca8364314512c33e4ae22d6 2024-11-13T05:55:17,603 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 200f36be3ca8364314512c33e4ae22d6: Running coprocessor pre-open hook at 1731477317579Writing region info on filesystem at 1731477317579Initializing all the Stores at 1731477317581 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317581Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317581Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317581Cleaning up temporary data from old regions at 1731477317595 (+14 ms)Running coprocessor post-open hooks at 1731477317603 (+8 ms)Region opened successfully at 1731477317603 2024-11-13T05:55:17,605 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6., pid=34, masterSystemTime=1731477317564 2024-11-13T05:55:17,608 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6. 2024-11-13T05:55:17,609 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6. 2024-11-13T05:55:17,609 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046. 2024-11-13T05:55:17,609 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 2810327548325125e764254e87017046, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-13T05:55:17,609 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=200f36be3ca8364314512c33e4ae22d6, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:17,609 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2810327548325125e764254e87017046 2024-11-13T05:55:17,610 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,610 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 2810327548325125e764254e87017046 2024-11-13T05:55:17,610 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 2810327548325125e764254e87017046 2024-11-13T05:55:17,613 INFO [StoreOpener-2810327548325125e764254e87017046-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 2810327548325125e764254e87017046 2024-11-13T05:55:17,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 200f36be3ca8364314512c33e4ae22d6, server=bc386b7bf2fe,45383,1731477314448 because future has completed 2024-11-13T05:55:17,617 INFO [StoreOpener-2810327548325125e764254e87017046-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2810327548325125e764254e87017046 columnFamilyName colfamily11 2024-11-13T05:55:17,618 DEBUG [StoreOpener-2810327548325125e764254e87017046-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,619 INFO [StoreOpener-2810327548325125e764254e87017046-1 {}] regionserver.HStore(327): Store=2810327548325125e764254e87017046/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,619 INFO [StoreOpener-2810327548325125e764254e87017046-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 2810327548325125e764254e87017046 2024-11-13T05:55:17,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=6 2024-11-13T05:55:17,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 200f36be3ca8364314512c33e4ae22d6, server=bc386b7bf2fe,45383,1731477314448 in 180 msec 2024-11-13T05:55:17,622 INFO [StoreOpener-2810327548325125e764254e87017046-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2810327548325125e764254e87017046 columnFamilyName colfamily21 2024-11-13T05:55:17,622 DEBUG [StoreOpener-2810327548325125e764254e87017046-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,623 INFO [StoreOpener-2810327548325125e764254e87017046-1 {}] regionserver.HStore(327): Store=2810327548325125e764254e87017046/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,623 INFO [StoreOpener-2810327548325125e764254e87017046-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 2810327548325125e764254e87017046 2024-11-13T05:55:17,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=200f36be3ca8364314512c33e4ae22d6, ASSIGN in 387 msec 2024-11-13T05:55:17,626 INFO [StoreOpener-2810327548325125e764254e87017046-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2810327548325125e764254e87017046 columnFamilyName colfamily31 2024-11-13T05:55:17,626 DEBUG [StoreOpener-2810327548325125e764254e87017046-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,627 INFO [StoreOpener-2810327548325125e764254e87017046-1 {}] regionserver.HStore(327): Store=2810327548325125e764254e87017046/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,627 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 2810327548325125e764254e87017046 2024-11-13T05:55:17,628 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2810327548325125e764254e87017046 2024-11-13T05:55:17,629 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2810327548325125e764254e87017046 2024-11-13T05:55:17,629 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T05:55:17,630 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 2810327548325125e764254e87017046 2024-11-13T05:55:17,631 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 2810327548325125e764254e87017046 2024-11-13T05:55:17,631 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39913, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:55:17,631 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,633 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 2810327548325125e764254e87017046 2024-11-13T05:55:17,636 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2810327548325125e764254e87017046/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,636 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e. 2024-11-13T05:55:17,636 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => a840e8f158b2eaf5c08595bd38db426e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-13T05:55:17,637 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true a840e8f158b2eaf5c08595bd38db426e 2024-11-13T05:55:17,637 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,637 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for a840e8f158b2eaf5c08595bd38db426e 2024-11-13T05:55:17,637 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 2810327548325125e764254e87017046; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68833857, jitterRate=0.02570439875125885}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,637 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for a840e8f158b2eaf5c08595bd38db426e 2024-11-13T05:55:17,637 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2810327548325125e764254e87017046 2024-11-13T05:55:17,638 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 2810327548325125e764254e87017046: Running coprocessor pre-open hook at 1731477317610Writing region info on filesystem at 1731477317610Initializing all the Stores at 1731477317612 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317612Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317612Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317612Cleaning up temporary data from old regions at 1731477317631 (+19 ms)Running coprocessor post-open hooks at 1731477317637 (+6 ms)Region opened successfully at 1731477317638 (+1 ms) 2024-11-13T05:55:17,639 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046., pid=38, masterSystemTime=1731477317564 2024-11-13T05:55:17,640 INFO [StoreOpener-a840e8f158b2eaf5c08595bd38db426e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region a840e8f158b2eaf5c08595bd38db426e 2024-11-13T05:55:17,642 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046. 2024-11-13T05:55:17,642 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046. 2024-11-13T05:55:17,642 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c. 2024-11-13T05:55:17,643 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 408dfc042fd5f2d964a15b01fd44966c, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-13T05:55:17,643 INFO [StoreOpener-a840e8f158b2eaf5c08595bd38db426e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a840e8f158b2eaf5c08595bd38db426e columnFamilyName colfamily11 2024-11-13T05:55:17,643 DEBUG [StoreOpener-a840e8f158b2eaf5c08595bd38db426e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,643 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 408dfc042fd5f2d964a15b01fd44966c 2024-11-13T05:55:17,643 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,643 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 408dfc042fd5f2d964a15b01fd44966c 2024-11-13T05:55:17,643 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=2810327548325125e764254e87017046, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:17,644 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 408dfc042fd5f2d964a15b01fd44966c 2024-11-13T05:55:17,644 INFO [StoreOpener-a840e8f158b2eaf5c08595bd38db426e-1 {}] regionserver.HStore(327): Store=a840e8f158b2eaf5c08595bd38db426e/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,645 INFO [StoreOpener-a840e8f158b2eaf5c08595bd38db426e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region a840e8f158b2eaf5c08595bd38db426e 2024-11-13T05:55:17,647 INFO [StoreOpener-a840e8f158b2eaf5c08595bd38db426e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a840e8f158b2eaf5c08595bd38db426e columnFamilyName colfamily21 2024-11-13T05:55:17,648 DEBUG [StoreOpener-a840e8f158b2eaf5c08595bd38db426e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2810327548325125e764254e87017046, server=bc386b7bf2fe,45383,1731477314448 because future has completed 2024-11-13T05:55:17,649 INFO [StoreOpener-408dfc042fd5f2d964a15b01fd44966c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 408dfc042fd5f2d964a15b01fd44966c 2024-11-13T05:55:17,649 INFO [StoreOpener-a840e8f158b2eaf5c08595bd38db426e-1 {}] regionserver.HStore(327): Store=a840e8f158b2eaf5c08595bd38db426e/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,649 INFO [StoreOpener-a840e8f158b2eaf5c08595bd38db426e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region a840e8f158b2eaf5c08595bd38db426e 2024-11-13T05:55:17,651 INFO [StoreOpener-a840e8f158b2eaf5c08595bd38db426e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a840e8f158b2eaf5c08595bd38db426e columnFamilyName colfamily31 2024-11-13T05:55:17,651 INFO [StoreOpener-408dfc042fd5f2d964a15b01fd44966c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 408dfc042fd5f2d964a15b01fd44966c columnFamilyName colfamily11 2024-11-13T05:55:17,652 DEBUG [StoreOpener-a840e8f158b2eaf5c08595bd38db426e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,652 DEBUG [StoreOpener-408dfc042fd5f2d964a15b01fd44966c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,652 INFO [StoreOpener-a840e8f158b2eaf5c08595bd38db426e-1 {}] regionserver.HStore(327): Store=a840e8f158b2eaf5c08595bd38db426e/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,652 INFO [StoreOpener-408dfc042fd5f2d964a15b01fd44966c-1 {}] regionserver.HStore(327): Store=408dfc042fd5f2d964a15b01fd44966c/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,653 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for a840e8f158b2eaf5c08595bd38db426e 2024-11-13T05:55:17,653 INFO [StoreOpener-408dfc042fd5f2d964a15b01fd44966c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 408dfc042fd5f2d964a15b01fd44966c 2024-11-13T05:55:17,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=24 2024-11-13T05:55:17,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 2810327548325125e764254e87017046, server=bc386b7bf2fe,45383,1731477314448 in 190 msec 2024-11-13T05:55:17,654 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a840e8f158b2eaf5c08595bd38db426e 2024-11-13T05:55:17,655 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a840e8f158b2eaf5c08595bd38db426e 2024-11-13T05:55:17,655 INFO [StoreOpener-408dfc042fd5f2d964a15b01fd44966c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 408dfc042fd5f2d964a15b01fd44966c columnFamilyName colfamily21 2024-11-13T05:55:17,655 DEBUG [StoreOpener-408dfc042fd5f2d964a15b01fd44966c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,656 INFO [StoreOpener-408dfc042fd5f2d964a15b01fd44966c-1 {}] regionserver.HStore(327): Store=408dfc042fd5f2d964a15b01fd44966c/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,656 INFO [StoreOpener-408dfc042fd5f2d964a15b01fd44966c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 408dfc042fd5f2d964a15b01fd44966c 2024-11-13T05:55:17,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2810327548325125e764254e87017046, ASSIGN in 421 msec 2024-11-13T05:55:17,657 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for a840e8f158b2eaf5c08595bd38db426e 2024-11-13T05:55:17,657 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for a840e8f158b2eaf5c08595bd38db426e 2024-11-13T05:55:17,658 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,658 INFO [StoreOpener-408dfc042fd5f2d964a15b01fd44966c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 408dfc042fd5f2d964a15b01fd44966c columnFamilyName colfamily31 2024-11-13T05:55:17,658 DEBUG [StoreOpener-408dfc042fd5f2d964a15b01fd44966c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,659 INFO [StoreOpener-408dfc042fd5f2d964a15b01fd44966c-1 {}] regionserver.HStore(327): Store=408dfc042fd5f2d964a15b01fd44966c/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,659 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 408dfc042fd5f2d964a15b01fd44966c 2024-11-13T05:55:17,660 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for a840e8f158b2eaf5c08595bd38db426e 2024-11-13T05:55:17,660 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408dfc042fd5f2d964a15b01fd44966c 2024-11-13T05:55:17,660 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408dfc042fd5f2d964a15b01fd44966c 2024-11-13T05:55:17,662 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 408dfc042fd5f2d964a15b01fd44966c 2024-11-13T05:55:17,662 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 408dfc042fd5f2d964a15b01fd44966c 2024-11-13T05:55:17,663 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a840e8f158b2eaf5c08595bd38db426e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,663 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,663 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened a840e8f158b2eaf5c08595bd38db426e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72046240, jitterRate=0.07357263565063477}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,664 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a840e8f158b2eaf5c08595bd38db426e 2024-11-13T05:55:17,664 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for a840e8f158b2eaf5c08595bd38db426e: Running coprocessor pre-open hook at 1731477317638Writing region info on filesystem at 1731477317638Initializing all the Stores at 1731477317639 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317639Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317640 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317640Cleaning up temporary data from old regions at 1731477317657 (+17 ms)Running coprocessor post-open hooks at 1731477317664 (+7 ms)Region opened successfully at 1731477317664 2024-11-13T05:55:17,665 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 408dfc042fd5f2d964a15b01fd44966c 2024-11-13T05:55:17,665 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e., pid=56, masterSystemTime=1731477317628 2024-11-13T05:55:17,668 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408dfc042fd5f2d964a15b01fd44966c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,668 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e. 2024-11-13T05:55:17,669 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e. 2024-11-13T05:55:17,669 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131. 2024-11-13T05:55:17,669 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 43fd53cccd0bd76519c89a78d6bf6131, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-13T05:55:17,669 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 43fd53cccd0bd76519c89a78d6bf6131 2024-11-13T05:55:17,670 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,669 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 408dfc042fd5f2d964a15b01fd44966c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67118761, jitterRate=1.4747679233551025E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,670 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 408dfc042fd5f2d964a15b01fd44966c 2024-11-13T05:55:17,670 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 43fd53cccd0bd76519c89a78d6bf6131 2024-11-13T05:55:17,670 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 408dfc042fd5f2d964a15b01fd44966c: Running coprocessor pre-open hook at 1731477317644Writing region info on filesystem at 1731477317644Initializing all the Stores at 1731477317648 (+4 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317648Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317648Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317648Cleaning up temporary data from old regions at 1731477317662 (+14 ms)Running coprocessor post-open hooks at 1731477317670 (+8 ms)Region opened successfully at 1731477317670 2024-11-13T05:55:17,670 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 43fd53cccd0bd76519c89a78d6bf6131 2024-11-13T05:55:17,670 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=a840e8f158b2eaf5c08595bd38db426e, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:55:17,671 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c., pid=32, masterSystemTime=1731477317564 2024-11-13T05:55:17,672 INFO [StoreOpener-43fd53cccd0bd76519c89a78d6bf6131-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 43fd53cccd0bd76519c89a78d6bf6131 2024-11-13T05:55:17,674 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure a840e8f158b2eaf5c08595bd38db426e, server=bc386b7bf2fe,40327,1731477314499 because future has completed 2024-11-13T05:55:17,674 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c. 2024-11-13T05:55:17,674 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c. 2024-11-13T05:55:17,674 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478. 2024-11-13T05:55:17,674 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 90b03edd8043eabbad2ddbd32409c478, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-13T05:55:17,675 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 90b03edd8043eabbad2ddbd32409c478 2024-11-13T05:55:17,675 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,675 INFO [StoreOpener-43fd53cccd0bd76519c89a78d6bf6131-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43fd53cccd0bd76519c89a78d6bf6131 columnFamilyName colfamily11 2024-11-13T05:55:17,675 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 90b03edd8043eabbad2ddbd32409c478 2024-11-13T05:55:17,675 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=408dfc042fd5f2d964a15b01fd44966c, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:17,675 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 90b03edd8043eabbad2ddbd32409c478 2024-11-13T05:55:17,675 DEBUG [StoreOpener-43fd53cccd0bd76519c89a78d6bf6131-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,676 INFO [StoreOpener-43fd53cccd0bd76519c89a78d6bf6131-1 {}] regionserver.HStore(327): Store=43fd53cccd0bd76519c89a78d6bf6131/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,676 INFO [StoreOpener-43fd53cccd0bd76519c89a78d6bf6131-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 43fd53cccd0bd76519c89a78d6bf6131 2024-11-13T05:55:17,678 INFO [StoreOpener-90b03edd8043eabbad2ddbd32409c478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 90b03edd8043eabbad2ddbd32409c478 2024-11-13T05:55:17,678 INFO [StoreOpener-43fd53cccd0bd76519c89a78d6bf6131-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43fd53cccd0bd76519c89a78d6bf6131 columnFamilyName colfamily21 2024-11-13T05:55:17,679 DEBUG [StoreOpener-43fd53cccd0bd76519c89a78d6bf6131-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 408dfc042fd5f2d964a15b01fd44966c, server=bc386b7bf2fe,45383,1731477314448 because future has completed 2024-11-13T05:55:17,679 INFO [StoreOpener-43fd53cccd0bd76519c89a78d6bf6131-1 {}] regionserver.HStore(327): Store=43fd53cccd0bd76519c89a78d6bf6131/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,679 INFO [StoreOpener-43fd53cccd0bd76519c89a78d6bf6131-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 43fd53cccd0bd76519c89a78d6bf6131 2024-11-13T05:55:17,680 INFO [StoreOpener-90b03edd8043eabbad2ddbd32409c478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90b03edd8043eabbad2ddbd32409c478 columnFamilyName colfamily11 2024-11-13T05:55:17,680 DEBUG [StoreOpener-90b03edd8043eabbad2ddbd32409c478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,681 INFO [StoreOpener-90b03edd8043eabbad2ddbd32409c478-1 {}] regionserver.HStore(327): Store=90b03edd8043eabbad2ddbd32409c478/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,681 INFO [StoreOpener-90b03edd8043eabbad2ddbd32409c478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 90b03edd8043eabbad2ddbd32409c478 2024-11-13T05:55:17,682 INFO [StoreOpener-43fd53cccd0bd76519c89a78d6bf6131-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43fd53cccd0bd76519c89a78d6bf6131 columnFamilyName colfamily31 2024-11-13T05:55:17,682 DEBUG [StoreOpener-43fd53cccd0bd76519c89a78d6bf6131-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,683 INFO [StoreOpener-90b03edd8043eabbad2ddbd32409c478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90b03edd8043eabbad2ddbd32409c478 columnFamilyName colfamily21 2024-11-13T05:55:17,683 INFO [StoreOpener-43fd53cccd0bd76519c89a78d6bf6131-1 {}] regionserver.HStore(327): Store=43fd53cccd0bd76519c89a78d6bf6131/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,683 DEBUG [StoreOpener-90b03edd8043eabbad2ddbd32409c478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=7 2024-11-13T05:55:17,683 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 43fd53cccd0bd76519c89a78d6bf6131 2024-11-13T05:55:17,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure a840e8f158b2eaf5c08595bd38db426e, server=bc386b7bf2fe,40327,1731477314499 in 108 msec 2024-11-13T05:55:17,684 INFO [StoreOpener-90b03edd8043eabbad2ddbd32409c478-1 {}] regionserver.HStore(327): Store=90b03edd8043eabbad2ddbd32409c478/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,684 INFO [StoreOpener-90b03edd8043eabbad2ddbd32409c478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 90b03edd8043eabbad2ddbd32409c478 2024-11-13T05:55:17,685 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43fd53cccd0bd76519c89a78d6bf6131 2024-11-13T05:55:17,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a840e8f158b2eaf5c08595bd38db426e, ASSIGN in 451 msec 2024-11-13T05:55:17,687 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a. 2024-11-13T05:55:17,687 INFO [StoreOpener-90b03edd8043eabbad2ddbd32409c478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90b03edd8043eabbad2ddbd32409c478 columnFamilyName colfamily31 2024-11-13T05:55:17,687 DEBUG [StoreOpener-90b03edd8043eabbad2ddbd32409c478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,687 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 5ff24ce91fac66115879ac4ec57a346a, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-13T05:55:17,688 INFO [StoreOpener-90b03edd8043eabbad2ddbd32409c478-1 {}] regionserver.HStore(327): Store=90b03edd8043eabbad2ddbd32409c478/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,688 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43fd53cccd0bd76519c89a78d6bf6131 2024-11-13T05:55:17,688 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 90b03edd8043eabbad2ddbd32409c478 2024-11-13T05:55:17,688 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:55:17,689 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,689 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:55:17,689 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:55:17,690 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/90b03edd8043eabbad2ddbd32409c478 2024-11-13T05:55:17,690 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 43fd53cccd0bd76519c89a78d6bf6131 2024-11-13T05:55:17,690 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 43fd53cccd0bd76519c89a78d6bf6131 2024-11-13T05:55:17,690 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/90b03edd8043eabbad2ddbd32409c478 2024-11-13T05:55:17,691 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,692 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 90b03edd8043eabbad2ddbd32409c478 2024-11-13T05:55:17,692 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 90b03edd8043eabbad2ddbd32409c478 2024-11-13T05:55:17,692 INFO [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:55:17,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=21 2024-11-13T05:55:17,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 408dfc042fd5f2d964a15b01fd44966c, server=bc386b7bf2fe,45383,1731477314448 in 271 msec 2024-11-13T05:55:17,693 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,693 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 43fd53cccd0bd76519c89a78d6bf6131 2024-11-13T05:55:17,695 INFO [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ff24ce91fac66115879ac4ec57a346a columnFamilyName colfamily11 2024-11-13T05:55:17,696 DEBUG [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,696 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 90b03edd8043eabbad2ddbd32409c478 2024-11-13T05:55:17,696 INFO [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] regionserver.HStore(327): Store=5ff24ce91fac66115879ac4ec57a346a/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=408dfc042fd5f2d964a15b01fd44966c, ASSIGN in 460 msec 2024-11-13T05:55:17,697 INFO [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:55:17,697 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43fd53cccd0bd76519c89a78d6bf6131/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,698 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 43fd53cccd0bd76519c89a78d6bf6131; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69108679, jitterRate=0.02979956567287445}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,698 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 43fd53cccd0bd76519c89a78d6bf6131 2024-11-13T05:55:17,698 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 43fd53cccd0bd76519c89a78d6bf6131: Running coprocessor pre-open hook at 1731477317670Writing region info on filesystem at 1731477317670Initializing all the Stores at 1731477317672 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317672Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317672Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317672Cleaning up temporary data from old regions at 1731477317690 (+18 ms)Running coprocessor post-open hooks at 1731477317698 (+8 ms)Region opened successfully at 1731477317698 2024-11-13T05:55:17,699 INFO [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ff24ce91fac66115879ac4ec57a346a columnFamilyName colfamily21 2024-11-13T05:55:17,699 DEBUG [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,700 INFO [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] regionserver.HStore(327): Store=5ff24ce91fac66115879ac4ec57a346a/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,700 INFO [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:55:17,700 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131., pid=44, masterSystemTime=1731477317628 2024-11-13T05:55:17,701 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/90b03edd8043eabbad2ddbd32409c478/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,702 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 90b03edd8043eabbad2ddbd32409c478; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74211221, jitterRate=0.10583336651325226}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,702 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 90b03edd8043eabbad2ddbd32409c478 2024-11-13T05:55:17,702 INFO [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ff24ce91fac66115879ac4ec57a346a columnFamilyName colfamily31 2024-11-13T05:55:17,702 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 90b03edd8043eabbad2ddbd32409c478: Running coprocessor pre-open hook at 1731477317675Writing region info on filesystem at 1731477317675Initializing all the Stores at 1731477317677 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317677Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317678 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317678Cleaning up temporary data from old regions at 1731477317692 (+14 ms)Running coprocessor post-open hooks at 1731477317702 (+10 ms)Region opened successfully at 1731477317702 2024-11-13T05:55:17,702 DEBUG [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,703 INFO [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] regionserver.HStore(327): Store=5ff24ce91fac66115879ac4ec57a346a/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,703 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:55:17,704 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478., pid=31, masterSystemTime=1731477317564 2024-11-13T05:55:17,704 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131. 2024-11-13T05:55:17,705 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131. 2024-11-13T05:55:17,705 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70. 2024-11-13T05:55:17,705 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 10c268d6da33b0a4590a478c0f32fd70, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-13T05:55:17,705 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 10c268d6da33b0a4590a478c0f32fd70 2024-11-13T05:55:17,705 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,705 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:55:17,705 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=43fd53cccd0bd76519c89a78d6bf6131, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:55:17,706 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 10c268d6da33b0a4590a478c0f32fd70 2024-11-13T05:55:17,706 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 10c268d6da33b0a4590a478c0f32fd70 2024-11-13T05:55:17,707 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:55:17,707 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478. 2024-11-13T05:55:17,707 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478. 2024-11-13T05:55:17,708 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c. 2024-11-13T05:55:17,708 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 408b59e776ffab17a3ee0e8fb465c67c, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-13T05:55:17,708 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 408b59e776ffab17a3ee0e8fb465c67c 2024-11-13T05:55:17,708 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,709 INFO [StoreOpener-10c268d6da33b0a4590a478c0f32fd70-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 10c268d6da33b0a4590a478c0f32fd70 2024-11-13T05:55:17,709 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 408b59e776ffab17a3ee0e8fb465c67c 2024-11-13T05:55:17,709 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 408b59e776ffab17a3ee0e8fb465c67c 2024-11-13T05:55:17,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43fd53cccd0bd76519c89a78d6bf6131, server=bc386b7bf2fe,40327,1731477314499 because future has completed 2024-11-13T05:55:17,709 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:55:17,709 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:55:17,710 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=90b03edd8043eabbad2ddbd32409c478, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:17,711 INFO [StoreOpener-10c268d6da33b0a4590a478c0f32fd70-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10c268d6da33b0a4590a478c0f32fd70 columnFamilyName colfamily11 2024-11-13T05:55:17,711 INFO [StoreOpener-408b59e776ffab17a3ee0e8fb465c67c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 408b59e776ffab17a3ee0e8fb465c67c 2024-11-13T05:55:17,711 DEBUG [StoreOpener-10c268d6da33b0a4590a478c0f32fd70-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,711 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,712 INFO [StoreOpener-10c268d6da33b0a4590a478c0f32fd70-1 {}] regionserver.HStore(327): Store=10c268d6da33b0a4590a478c0f32fd70/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,713 INFO [StoreOpener-10c268d6da33b0a4590a478c0f32fd70-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 10c268d6da33b0a4590a478c0f32fd70 2024-11-13T05:55:17,713 INFO [StoreOpener-408b59e776ffab17a3ee0e8fb465c67c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 408b59e776ffab17a3ee0e8fb465c67c columnFamilyName colfamily11 2024-11-13T05:55:17,713 DEBUG [StoreOpener-408b59e776ffab17a3ee0e8fb465c67c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90b03edd8043eabbad2ddbd32409c478, server=bc386b7bf2fe,45383,1731477314448 because future has completed 2024-11-13T05:55:17,714 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:55:17,714 INFO [StoreOpener-408b59e776ffab17a3ee0e8fb465c67c-1 {}] regionserver.HStore(327): Store=408b59e776ffab17a3ee0e8fb465c67c/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,715 INFO [StoreOpener-408b59e776ffab17a3ee0e8fb465c67c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 408b59e776ffab17a3ee0e8fb465c67c 2024-11-13T05:55:17,715 INFO [StoreOpener-10c268d6da33b0a4590a478c0f32fd70-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10c268d6da33b0a4590a478c0f32fd70 columnFamilyName colfamily21 2024-11-13T05:55:17,716 DEBUG [StoreOpener-10c268d6da33b0a4590a478c0f32fd70-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,717 INFO [StoreOpener-408b59e776ffab17a3ee0e8fb465c67c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 408b59e776ffab17a3ee0e8fb465c67c columnFamilyName colfamily21 2024-11-13T05:55:17,718 DEBUG [StoreOpener-408b59e776ffab17a3ee0e8fb465c67c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,718 INFO [StoreOpener-10c268d6da33b0a4590a478c0f32fd70-1 {}] regionserver.HStore(327): Store=10c268d6da33b0a4590a478c0f32fd70/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,718 INFO [StoreOpener-10c268d6da33b0a4590a478c0f32fd70-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 10c268d6da33b0a4590a478c0f32fd70 2024-11-13T05:55:17,718 INFO [StoreOpener-408b59e776ffab17a3ee0e8fb465c67c-1 {}] regionserver.HStore(327): Store=408b59e776ffab17a3ee0e8fb465c67c/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,719 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,719 INFO [StoreOpener-408b59e776ffab17a3ee0e8fb465c67c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 408b59e776ffab17a3ee0e8fb465c67c 2024-11-13T05:55:17,720 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 5ff24ce91fac66115879ac4ec57a346a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74950836, jitterRate=0.11685448884963989}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,720 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:55:17,720 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 5ff24ce91fac66115879ac4ec57a346a: Running coprocessor pre-open hook at 1731477317689Writing region info on filesystem at 1731477317689Initializing all the Stores at 1731477317690 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317691 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317691Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317691Cleaning up temporary data from old regions at 1731477317709 (+18 ms)Running coprocessor post-open hooks at 1731477317720 (+11 ms)Region opened successfully at 1731477317720 2024-11-13T05:55:17,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=19 2024-11-13T05:55:17,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 43fd53cccd0bd76519c89a78d6bf6131, server=bc386b7bf2fe,40327,1731477314499 in 223 msec 2024-11-13T05:55:17,721 INFO [StoreOpener-10c268d6da33b0a4590a478c0f32fd70-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10c268d6da33b0a4590a478c0f32fd70 columnFamilyName colfamily31 2024-11-13T05:55:17,721 DEBUG [StoreOpener-10c268d6da33b0a4590a478c0f32fd70-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,722 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a., pid=52, masterSystemTime=1731477317681 2024-11-13T05:55:17,723 INFO [StoreOpener-408b59e776ffab17a3ee0e8fb465c67c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 408b59e776ffab17a3ee0e8fb465c67c columnFamilyName colfamily31 2024-11-13T05:55:17,723 DEBUG [StoreOpener-408b59e776ffab17a3ee0e8fb465c67c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=43fd53cccd0bd76519c89a78d6bf6131, ASSIGN in 488 msec 2024-11-13T05:55:17,724 INFO [StoreOpener-10c268d6da33b0a4590a478c0f32fd70-1 {}] regionserver.HStore(327): Store=10c268d6da33b0a4590a478c0f32fd70/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,724 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 10c268d6da33b0a4590a478c0f32fd70 2024-11-13T05:55:17,725 INFO [StoreOpener-408b59e776ffab17a3ee0e8fb465c67c-1 {}] regionserver.HStore(327): Store=408b59e776ffab17a3ee0e8fb465c67c/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,725 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 408b59e776ffab17a3ee0e8fb465c67c 2024-11-13T05:55:17,726 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10c268d6da33b0a4590a478c0f32fd70 2024-11-13T05:55:17,727 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10c268d6da33b0a4590a478c0f32fd70 2024-11-13T05:55:17,727 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408b59e776ffab17a3ee0e8fb465c67c 2024-11-13T05:55:17,729 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 10c268d6da33b0a4590a478c0f32fd70 2024-11-13T05:55:17,729 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408b59e776ffab17a3ee0e8fb465c67c 2024-11-13T05:55:17,729 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 10c268d6da33b0a4590a478c0f32fd70 2024-11-13T05:55:17,730 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a. 2024-11-13T05:55:17,730 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a. 2024-11-13T05:55:17,730 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca. 2024-11-13T05:55:17,730 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,730 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 9afb90ba9af49bb3547d4438986e1fca, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-13T05:55:17,731 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:55:17,731 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,731 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:55:17,731 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 408b59e776ffab17a3ee0e8fb465c67c 2024-11-13T05:55:17,731 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 408b59e776ffab17a3ee0e8fb465c67c 2024-11-13T05:55:17,731 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:55:17,732 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 10c268d6da33b0a4590a478c0f32fd70 2024-11-13T05:55:17,732 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-13T05:55:17,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 90b03edd8043eabbad2ddbd32409c478, server=bc386b7bf2fe,45383,1731477314448 in 320 msec 2024-11-13T05:55:17,734 INFO [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:55:17,736 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10c268d6da33b0a4590a478c0f32fd70/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,736 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 408b59e776ffab17a3ee0e8fb465c67c 2024-11-13T05:55:17,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=90b03edd8043eabbad2ddbd32409c478, ASSIGN in 500 msec 2024-11-13T05:55:17,737 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 10c268d6da33b0a4590a478c0f32fd70; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67026818, jitterRate=-0.0012225806713104248}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,737 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 10c268d6da33b0a4590a478c0f32fd70 2024-11-13T05:55:17,737 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 10c268d6da33b0a4590a478c0f32fd70: Running coprocessor pre-open hook at 1731477317706Writing region info on filesystem at 1731477317706Initializing all the Stores at 1731477317707 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317707Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317708 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317708Cleaning up temporary data from old regions at 1731477317729 (+21 ms)Running coprocessor post-open hooks at 1731477317737 (+8 ms)Region opened successfully at 1731477317737 2024-11-13T05:55:17,738 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=5ff24ce91fac66115879ac4ec57a346a, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:17,739 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70., pid=42, masterSystemTime=1731477317628 2024-11-13T05:55:17,739 INFO [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9afb90ba9af49bb3547d4438986e1fca columnFamilyName colfamily11 2024-11-13T05:55:17,739 DEBUG [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,741 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408b59e776ffab17a3ee0e8fb465c67c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,741 INFO [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] regionserver.HStore(327): Store=9afb90ba9af49bb3547d4438986e1fca/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,741 INFO [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:55:17,742 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ff24ce91fac66115879ac4ec57a346a, server=bc386b7bf2fe,38823,1731477314332 because future has completed 2024-11-13T05:55:17,742 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70. 2024-11-13T05:55:17,742 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70. 2024-11-13T05:55:17,742 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e. 2024-11-13T05:55:17,742 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 80710108d60e6f4a1b8975ccfe51e50e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-13T05:55:17,743 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 80710108d60e6f4a1b8975ccfe51e50e 2024-11-13T05:55:17,743 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,743 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 80710108d60e6f4a1b8975ccfe51e50e 2024-11-13T05:55:17,743 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=10c268d6da33b0a4590a478c0f32fd70, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:55:17,743 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 80710108d60e6f4a1b8975ccfe51e50e 2024-11-13T05:55:17,745 INFO [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9afb90ba9af49bb3547d4438986e1fca columnFamilyName colfamily21 2024-11-13T05:55:17,745 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 408b59e776ffab17a3ee0e8fb465c67c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67929206, jitterRate=0.012224048376083374}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,745 DEBUG [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,745 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 408b59e776ffab17a3ee0e8fb465c67c 2024-11-13T05:55:17,745 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 408b59e776ffab17a3ee0e8fb465c67c: Running coprocessor pre-open hook at 1731477317709Writing region info on filesystem at 1731477317709Initializing all the Stores at 1731477317710 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317710Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317711 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317711Cleaning up temporary data from old regions at 1731477317731 (+20 ms)Running coprocessor post-open hooks at 1731477317745 (+14 ms)Region opened successfully at 1731477317745 2024-11-13T05:55:17,745 INFO [StoreOpener-80710108d60e6f4a1b8975ccfe51e50e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 80710108d60e6f4a1b8975ccfe51e50e 2024-11-13T05:55:17,747 INFO [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] regionserver.HStore(327): Store=9afb90ba9af49bb3547d4438986e1fca/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,747 INFO [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:55:17,747 INFO [StoreOpener-80710108d60e6f4a1b8975ccfe51e50e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 80710108d60e6f4a1b8975ccfe51e50e columnFamilyName colfamily11 2024-11-13T05:55:17,747 DEBUG [StoreOpener-80710108d60e6f4a1b8975ccfe51e50e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,747 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10c268d6da33b0a4590a478c0f32fd70, server=bc386b7bf2fe,40327,1731477314499 because future has completed 2024-11-13T05:55:17,748 INFO [StoreOpener-80710108d60e6f4a1b8975ccfe51e50e-1 {}] regionserver.HStore(327): Store=80710108d60e6f4a1b8975ccfe51e50e/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,748 INFO [StoreOpener-80710108d60e6f4a1b8975ccfe51e50e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 80710108d60e6f4a1b8975ccfe51e50e 2024-11-13T05:55:17,749 INFO [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9afb90ba9af49bb3547d4438986e1fca columnFamilyName colfamily31 2024-11-13T05:55:17,750 DEBUG [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,750 INFO [StoreOpener-80710108d60e6f4a1b8975ccfe51e50e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 80710108d60e6f4a1b8975ccfe51e50e columnFamilyName colfamily21 2024-11-13T05:55:17,750 DEBUG [StoreOpener-80710108d60e6f4a1b8975ccfe51e50e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,750 INFO [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] regionserver.HStore(327): Store=9afb90ba9af49bb3547d4438986e1fca/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,751 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:55:17,751 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c., pid=35, masterSystemTime=1731477317564 2024-11-13T05:55:17,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=17 2024-11-13T05:55:17,753 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:55:17,753 INFO [StoreOpener-80710108d60e6f4a1b8975ccfe51e50e-1 {}] regionserver.HStore(327): Store=80710108d60e6f4a1b8975ccfe51e50e/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 5ff24ce91fac66115879ac4ec57a346a, server=bc386b7bf2fe,38823,1731477314332 in 195 msec 2024-11-13T05:55:17,753 INFO [StoreOpener-80710108d60e6f4a1b8975ccfe51e50e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 80710108d60e6f4a1b8975ccfe51e50e 2024-11-13T05:55:17,753 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:55:17,755 INFO [StoreOpener-80710108d60e6f4a1b8975ccfe51e50e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 80710108d60e6f4a1b8975ccfe51e50e columnFamilyName colfamily31 2024-11-13T05:55:17,755 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c. 2024-11-13T05:55:17,755 DEBUG [StoreOpener-80710108d60e6f4a1b8975ccfe51e50e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,755 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c. 2024-11-13T05:55:17,756 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:55:17,756 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:55:17,756 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b. 2024-11-13T05:55:17,756 INFO [StoreOpener-80710108d60e6f4a1b8975ccfe51e50e-1 {}] regionserver.HStore(327): Store=80710108d60e6f4a1b8975ccfe51e50e/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,756 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 80710108d60e6f4a1b8975ccfe51e50e 2024-11-13T05:55:17,756 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => f7c7452ce5410ec8dcf456adb9b7db6b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-13T05:55:17,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ff24ce91fac66115879ac4ec57a346a, ASSIGN in 520 msec 2024-11-13T05:55:17,756 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,757 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true f7c7452ce5410ec8dcf456adb9b7db6b 2024-11-13T05:55:17,757 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,757 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for f7c7452ce5410ec8dcf456adb9b7db6b 2024-11-13T05:55:17,757 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for f7c7452ce5410ec8dcf456adb9b7db6b 2024-11-13T05:55:17,758 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/80710108d60e6f4a1b8975ccfe51e50e 2024-11-13T05:55:17,758 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/80710108d60e6f4a1b8975ccfe51e50e 2024-11-13T05:55:17,759 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:55:17,759 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=408b59e776ffab17a3ee0e8fb465c67c, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:17,759 INFO [StoreOpener-f7c7452ce5410ec8dcf456adb9b7db6b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region f7c7452ce5410ec8dcf456adb9b7db6b 2024-11-13T05:55:17,760 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 80710108d60e6f4a1b8975ccfe51e50e 2024-11-13T05:55:17,760 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 80710108d60e6f4a1b8975ccfe51e50e 2024-11-13T05:55:17,761 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,762 INFO [StoreOpener-f7c7452ce5410ec8dcf456adb9b7db6b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7c7452ce5410ec8dcf456adb9b7db6b columnFamilyName colfamily11 2024-11-13T05:55:17,762 DEBUG [StoreOpener-f7c7452ce5410ec8dcf456adb9b7db6b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 408b59e776ffab17a3ee0e8fb465c67c, server=bc386b7bf2fe,45383,1731477314448 because future has completed 2024-11-13T05:55:17,763 INFO [StoreOpener-f7c7452ce5410ec8dcf456adb9b7db6b-1 {}] regionserver.HStore(327): Store=f7c7452ce5410ec8dcf456adb9b7db6b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,763 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 80710108d60e6f4a1b8975ccfe51e50e 2024-11-13T05:55:17,763 INFO [StoreOpener-f7c7452ce5410ec8dcf456adb9b7db6b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region f7c7452ce5410ec8dcf456adb9b7db6b 2024-11-13T05:55:17,765 INFO [StoreOpener-f7c7452ce5410ec8dcf456adb9b7db6b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7c7452ce5410ec8dcf456adb9b7db6b columnFamilyName colfamily21 2024-11-13T05:55:17,766 DEBUG [StoreOpener-f7c7452ce5410ec8dcf456adb9b7db6b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,767 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,767 INFO [StoreOpener-f7c7452ce5410ec8dcf456adb9b7db6b-1 {}] regionserver.HStore(327): Store=f7c7452ce5410ec8dcf456adb9b7db6b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,767 INFO [StoreOpener-f7c7452ce5410ec8dcf456adb9b7db6b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region f7c7452ce5410ec8dcf456adb9b7db6b 2024-11-13T05:55:17,768 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 9afb90ba9af49bb3547d4438986e1fca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71495624, jitterRate=0.06536781787872314}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,768 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:55:17,768 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 9afb90ba9af49bb3547d4438986e1fca: Running coprocessor pre-open hook at 1731477317732Writing region info on filesystem at 1731477317732Initializing all the Stores at 1731477317734 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317734Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317734Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317734Cleaning up temporary data from old regions at 1731477317756 (+22 ms)Running coprocessor post-open hooks at 1731477317768 (+12 ms)Region opened successfully at 1731477317768 2024-11-13T05:55:17,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=16 2024-11-13T05:55:17,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 10c268d6da33b0a4590a478c0f32fd70, server=bc386b7bf2fe,40327,1731477314499 in 281 msec 2024-11-13T05:55:17,769 INFO [StoreOpener-f7c7452ce5410ec8dcf456adb9b7db6b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7c7452ce5410ec8dcf456adb9b7db6b columnFamilyName colfamily31 2024-11-13T05:55:17,769 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca., pid=53, masterSystemTime=1731477317681 2024-11-13T05:55:17,769 DEBUG [StoreOpener-f7c7452ce5410ec8dcf456adb9b7db6b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,771 INFO [StoreOpener-f7c7452ce5410ec8dcf456adb9b7db6b-1 {}] regionserver.HStore(327): Store=f7c7452ce5410ec8dcf456adb9b7db6b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,771 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for f7c7452ce5410ec8dcf456adb9b7db6b 2024-11-13T05:55:17,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=10c268d6da33b0a4590a478c0f32fd70, ASSIGN in 536 msec 2024-11-13T05:55:17,773 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f7c7452ce5410ec8dcf456adb9b7db6b 2024-11-13T05:55:17,773 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca. 2024-11-13T05:55:17,773 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca. 2024-11-13T05:55:17,773 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f7c7452ce5410ec8dcf456adb9b7db6b 2024-11-13T05:55:17,774 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225. 2024-11-13T05:55:17,774 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => a65241dbe6c2863f31a0138a73130225, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-13T05:55:17,775 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true a65241dbe6c2863f31a0138a73130225 2024-11-13T05:55:17,775 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,775 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=9afb90ba9af49bb3547d4438986e1fca, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:17,775 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for a65241dbe6c2863f31a0138a73130225 2024-11-13T05:55:17,775 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for a65241dbe6c2863f31a0138a73130225 2024-11-13T05:55:17,776 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for f7c7452ce5410ec8dcf456adb9b7db6b 2024-11-13T05:55:17,776 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for f7c7452ce5410ec8dcf456adb9b7db6b 2024-11-13T05:55:17,777 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,779 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for f7c7452ce5410ec8dcf456adb9b7db6b 2024-11-13T05:55:17,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9afb90ba9af49bb3547d4438986e1fca, server=bc386b7bf2fe,38823,1731477314332 because future has completed 2024-11-13T05:55:17,782 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/80710108d60e6f4a1b8975ccfe51e50e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,783 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f7c7452ce5410ec8dcf456adb9b7db6b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,783 INFO [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region a65241dbe6c2863f31a0138a73130225 2024-11-13T05:55:17,784 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 80710108d60e6f4a1b8975ccfe51e50e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60476101, jitterRate=-0.09883587062358856}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,784 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 80710108d60e6f4a1b8975ccfe51e50e 2024-11-13T05:55:17,784 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 80710108d60e6f4a1b8975ccfe51e50e: Running coprocessor pre-open hook at 1731477317743Writing region info on filesystem at 1731477317743Initializing all the Stores at 1731477317745 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317745Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317745Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317745Cleaning up temporary data from old regions at 1731477317760 (+15 ms)Running coprocessor post-open hooks at 1731477317784 (+24 ms)Region opened successfully at 1731477317784 2024-11-13T05:55:17,784 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened f7c7452ce5410ec8dcf456adb9b7db6b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61111559, jitterRate=-0.08936680853366852}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,784 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f7c7452ce5410ec8dcf456adb9b7db6b 2024-11-13T05:55:17,784 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for f7c7452ce5410ec8dcf456adb9b7db6b: Running coprocessor pre-open hook at 1731477317757Writing region info on filesystem at 1731477317757Initializing all the Stores at 1731477317759 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317759Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317759Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317759Cleaning up temporary data from old regions at 1731477317776 (+17 ms)Running coprocessor post-open hooks at 1731477317784 (+8 ms)Region opened successfully at 1731477317784 2024-11-13T05:55:17,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=12 2024-11-13T05:55:17,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 408b59e776ffab17a3ee0e8fb465c67c, server=bc386b7bf2fe,45383,1731477314448 in 338 msec 2024-11-13T05:55:17,785 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e., pid=40, masterSystemTime=1731477317628 2024-11-13T05:55:17,785 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b., pid=37, masterSystemTime=1731477317564 2024-11-13T05:55:17,786 INFO [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a65241dbe6c2863f31a0138a73130225 columnFamilyName colfamily11 2024-11-13T05:55:17,786 DEBUG [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,787 INFO [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] regionserver.HStore(327): Store=a65241dbe6c2863f31a0138a73130225/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,787 INFO [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region a65241dbe6c2863f31a0138a73130225 2024-11-13T05:55:17,790 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b. 2024-11-13T05:55:17,790 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b. 2024-11-13T05:55:17,790 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b. 2024-11-13T05:55:17,790 INFO [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a65241dbe6c2863f31a0138a73130225 columnFamilyName colfamily21 2024-11-13T05:55:17,790 DEBUG [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,790 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 19321613b69704e42207868d450f785b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-13T05:55:17,791 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 19321613b69704e42207868d450f785b 2024-11-13T05:55:17,791 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,791 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 19321613b69704e42207868d450f785b 2024-11-13T05:55:17,791 INFO [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] regionserver.HStore(327): Store=a65241dbe6c2863f31a0138a73130225/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,791 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 19321613b69704e42207868d450f785b 2024-11-13T05:55:17,791 INFO [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region a65241dbe6c2863f31a0138a73130225 2024-11-13T05:55:17,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=408b59e776ffab17a3ee0e8fb465c67c, ASSIGN in 552 msec 2024-11-13T05:55:17,793 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e. 2024-11-13T05:55:17,793 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e. 2024-11-13T05:55:17,793 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94. 2024-11-13T05:55:17,793 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => e86ecaa5f83edede0f153db264078a94, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-13T05:55:17,794 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true e86ecaa5f83edede0f153db264078a94 2024-11-13T05:55:17,794 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,794 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for e86ecaa5f83edede0f153db264078a94 2024-11-13T05:55:17,794 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for e86ecaa5f83edede0f153db264078a94 2024-11-13T05:55:17,794 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=80710108d60e6f4a1b8975ccfe51e50e, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:55:17,795 INFO [StoreOpener-19321613b69704e42207868d450f785b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 19321613b69704e42207868d450f785b 2024-11-13T05:55:17,797 INFO [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a65241dbe6c2863f31a0138a73130225 columnFamilyName colfamily31 2024-11-13T05:55:17,797 DEBUG [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80710108d60e6f4a1b8975ccfe51e50e, server=bc386b7bf2fe,40327,1731477314499 because future has completed 2024-11-13T05:55:17,798 INFO [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] regionserver.HStore(327): Store=a65241dbe6c2863f31a0138a73130225/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,799 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for a65241dbe6c2863f31a0138a73130225 2024-11-13T05:55:17,799 INFO [StoreOpener-19321613b69704e42207868d450f785b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19321613b69704e42207868d450f785b columnFamilyName colfamily11 2024-11-13T05:55:17,799 DEBUG [StoreOpener-19321613b69704e42207868d450f785b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,800 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=f7c7452ce5410ec8dcf456adb9b7db6b, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:17,800 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225 2024-11-13T05:55:17,800 INFO [StoreOpener-19321613b69704e42207868d450f785b-1 {}] regionserver.HStore(327): Store=19321613b69704e42207868d450f785b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,800 INFO [StoreOpener-19321613b69704e42207868d450f785b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 19321613b69704e42207868d450f785b 2024-11-13T05:55:17,800 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225 2024-11-13T05:55:17,803 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for a65241dbe6c2863f31a0138a73130225 2024-11-13T05:55:17,803 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for a65241dbe6c2863f31a0138a73130225 2024-11-13T05:55:17,803 INFO [StoreOpener-19321613b69704e42207868d450f785b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19321613b69704e42207868d450f785b columnFamilyName colfamily21 2024-11-13T05:55:17,803 DEBUG [StoreOpener-19321613b69704e42207868d450f785b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,804 INFO [StoreOpener-19321613b69704e42207868d450f785b-1 {}] regionserver.HStore(327): Store=19321613b69704e42207868d450f785b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,804 INFO [StoreOpener-19321613b69704e42207868d450f785b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 19321613b69704e42207868d450f785b 2024-11-13T05:55:17,804 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7c7452ce5410ec8dcf456adb9b7db6b, server=bc386b7bf2fe,45383,1731477314448 because future has completed 2024-11-13T05:55:17,807 INFO [StoreOpener-e86ecaa5f83edede0f153db264078a94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region e86ecaa5f83edede0f153db264078a94 2024-11-13T05:55:17,807 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for a65241dbe6c2863f31a0138a73130225 2024-11-13T05:55:17,807 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=bc386b7bf2fe,40327,1731477314499, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=80710108d60e6f4a1b8975ccfe51e50e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T05:55:17,810 INFO [StoreOpener-19321613b69704e42207868d450f785b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19321613b69704e42207868d450f785b columnFamilyName colfamily31 2024-11-13T05:55:17,810 DEBUG [StoreOpener-19321613b69704e42207868d450f785b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,810 INFO [StoreOpener-e86ecaa5f83edede0f153db264078a94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e86ecaa5f83edede0f153db264078a94 columnFamilyName colfamily11 2024-11-13T05:55:17,811 DEBUG [StoreOpener-e86ecaa5f83edede0f153db264078a94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,811 INFO [StoreOpener-e86ecaa5f83edede0f153db264078a94-1 {}] regionserver.HStore(327): Store=e86ecaa5f83edede0f153db264078a94/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,811 INFO [StoreOpener-19321613b69704e42207868d450f785b-1 {}] regionserver.HStore(327): Store=19321613b69704e42207868d450f785b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,811 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 19321613b69704e42207868d450f785b 2024-11-13T05:55:17,812 INFO [StoreOpener-e86ecaa5f83edede0f153db264078a94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region e86ecaa5f83edede0f153db264078a94 2024-11-13T05:55:17,812 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/19321613b69704e42207868d450f785b 2024-11-13T05:55:17,813 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,813 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/19321613b69704e42207868d450f785b 2024-11-13T05:55:17,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=18 2024-11-13T05:55:17,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure f7c7452ce5410ec8dcf456adb9b7db6b, server=bc386b7bf2fe,45383,1731477314448 in 354 msec 2024-11-13T05:55:17,813 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened a65241dbe6c2863f31a0138a73130225; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65100171, jitterRate=-0.02993185818195343}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,814 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a65241dbe6c2863f31a0138a73130225 2024-11-13T05:55:17,814 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for a65241dbe6c2863f31a0138a73130225: Running coprocessor pre-open hook at 1731477317775Writing region info on filesystem at 1731477317775Initializing all the Stores at 1731477317777 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317777Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317782 (+5 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317782Cleaning up temporary data from old regions at 1731477317803 (+21 ms)Running coprocessor post-open hooks at 1731477317814 (+11 ms)Region opened successfully at 1731477317814 2024-11-13T05:55:17,815 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 19321613b69704e42207868d450f785b 2024-11-13T05:55:17,815 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 19321613b69704e42207868d450f785b 2024-11-13T05:55:17,815 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225., pid=54, masterSystemTime=1731477317681 2024-11-13T05:55:17,815 INFO [StoreOpener-e86ecaa5f83edede0f153db264078a94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e86ecaa5f83edede0f153db264078a94 columnFamilyName colfamily21 2024-11-13T05:55:17,815 DEBUG [StoreOpener-e86ecaa5f83edede0f153db264078a94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,816 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,816 INFO [StoreOpener-e86ecaa5f83edede0f153db264078a94-1 {}] regionserver.HStore(327): Store=e86ecaa5f83edede0f153db264078a94/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f7c7452ce5410ec8dcf456adb9b7db6b, ASSIGN in 581 msec 2024-11-13T05:55:17,817 INFO [StoreOpener-e86ecaa5f83edede0f153db264078a94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region e86ecaa5f83edede0f153db264078a94 2024-11-13T05:55:17,818 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 19321613b69704e42207868d450f785b 2024-11-13T05:55:17,819 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225. 2024-11-13T05:55:17,819 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225. 2024-11-13T05:55:17,819 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735. 2024-11-13T05:55:17,819 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 98cb12bd0a37bbbfee08e3459a85f735, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-13T05:55:17,820 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:55:17,820 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,820 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:55:17,820 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:55:17,821 INFO [StoreOpener-e86ecaa5f83edede0f153db264078a94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e86ecaa5f83edede0f153db264078a94 columnFamilyName colfamily31 2024-11-13T05:55:17,821 DEBUG [StoreOpener-e86ecaa5f83edede0f153db264078a94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,822 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/19321613b69704e42207868d450f785b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,822 INFO [StoreOpener-e86ecaa5f83edede0f153db264078a94-1 {}] regionserver.HStore(327): Store=e86ecaa5f83edede0f153db264078a94/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,823 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for e86ecaa5f83edede0f153db264078a94 2024-11-13T05:55:17,823 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 19321613b69704e42207868d450f785b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69014869, jitterRate=0.028401687741279602}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,823 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 19321613b69704e42207868d450f785b 2024-11-13T05:55:17,823 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 19321613b69704e42207868d450f785b: Running coprocessor pre-open hook at 1731477317791Writing region info on filesystem at 1731477317791Initializing all the Stores at 1731477317795 (+4 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317795Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317795Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317795Cleaning up temporary data from old regions at 1731477317815 (+20 ms)Running coprocessor post-open hooks at 1731477317823 (+8 ms)Region opened successfully at 1731477317823 2024-11-13T05:55:17,824 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e86ecaa5f83edede0f153db264078a94 2024-11-13T05:55:17,824 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b., pid=33, masterSystemTime=1731477317564 2024-11-13T05:55:17,824 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e86ecaa5f83edede0f153db264078a94 2024-11-13T05:55:17,826 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for e86ecaa5f83edede0f153db264078a94 2024-11-13T05:55:17,826 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for e86ecaa5f83edede0f153db264078a94 2024-11-13T05:55:17,827 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=28 2024-11-13T05:55:17,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 80710108d60e6f4a1b8975ccfe51e50e, server=bc386b7bf2fe,40327,1731477314499 in 346 msec 2024-11-13T05:55:17,827 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b. 2024-11-13T05:55:17,827 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b. 2024-11-13T05:55:17,828 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52. 2024-11-13T05:55:17,828 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => b540ac2e8c4575b424d481848c4b4e52, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-13T05:55:17,828 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true b540ac2e8c4575b424d481848c4b4e52 2024-11-13T05:55:17,828 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,829 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for b540ac2e8c4575b424d481848c4b4e52 2024-11-13T05:55:17,829 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for b540ac2e8c4575b424d481848c4b4e52 2024-11-13T05:55:17,829 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for e86ecaa5f83edede0f153db264078a94 2024-11-13T05:55:17,829 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=19321613b69704e42207868d450f785b, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:17,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 19321613b69704e42207868d450f785b, server=bc386b7bf2fe,45383,1731477314448 because future has completed 2024-11-13T05:55:17,835 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=bc386b7bf2fe,38823,1731477314332, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9afb90ba9af49bb3547d4438986e1fca. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T05:55:17,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=80710108d60e6f4a1b8975ccfe51e50e, ASSIGN in 601 msec 2024-11-13T05:55:17,839 INFO [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:55:17,839 INFO [StoreOpener-b540ac2e8c4575b424d481848c4b4e52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region b540ac2e8c4575b424d481848c4b4e52 2024-11-13T05:55:17,840 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e86ecaa5f83edede0f153db264078a94/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,840 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened e86ecaa5f83edede0f153db264078a94; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67264289, jitterRate=0.002316012978553772}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,841 INFO [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98cb12bd0a37bbbfee08e3459a85f735 columnFamilyName colfamily11 2024-11-13T05:55:17,841 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e86ecaa5f83edede0f153db264078a94 2024-11-13T05:55:17,841 DEBUG [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,841 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for e86ecaa5f83edede0f153db264078a94: Running coprocessor pre-open hook at 1731477317794Writing region info on filesystem at 1731477317794Initializing all the Stores at 1731477317796 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317796Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317806 (+10 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317806Cleaning up temporary data from old regions at 1731477317826 (+20 ms)Running coprocessor post-open hooks at 1731477317841 (+15 ms)Region opened successfully at 1731477317841 2024-11-13T05:55:17,842 INFO [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] regionserver.HStore(327): Store=98cb12bd0a37bbbfee08e3459a85f735/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,842 INFO [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:55:17,842 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94., pid=41, masterSystemTime=1731477317628 2024-11-13T05:55:17,843 INFO [StoreOpener-b540ac2e8c4575b424d481848c4b4e52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b540ac2e8c4575b424d481848c4b4e52 columnFamilyName colfamily11 2024-11-13T05:55:17,843 DEBUG [StoreOpener-b540ac2e8c4575b424d481848c4b4e52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,844 INFO [StoreOpener-b540ac2e8c4575b424d481848c4b4e52-1 {}] regionserver.HStore(327): Store=b540ac2e8c4575b424d481848c4b4e52/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,845 INFO [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98cb12bd0a37bbbfee08e3459a85f735 columnFamilyName colfamily21 2024-11-13T05:55:17,845 DEBUG [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=9 2024-11-13T05:55:17,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 19321613b69704e42207868d450f785b, server=bc386b7bf2fe,45383,1731477314448 in 410 msec 2024-11-13T05:55:17,846 INFO [StoreOpener-b540ac2e8c4575b424d481848c4b4e52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region b540ac2e8c4575b424d481848c4b4e52 2024-11-13T05:55:17,846 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94. 2024-11-13T05:55:17,846 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94. 2024-11-13T05:55:17,846 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc. 2024-11-13T05:55:17,846 INFO [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] regionserver.HStore(327): Store=98cb12bd0a37bbbfee08e3459a85f735/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,846 INFO [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:55:17,846 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 89e8c9d6d5af67c75803894f4aea8ecc, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-13T05:55:17,847 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 89e8c9d6d5af67c75803894f4aea8ecc 2024-11-13T05:55:17,847 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,847 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 89e8c9d6d5af67c75803894f4aea8ecc 2024-11-13T05:55:17,847 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 89e8c9d6d5af67c75803894f4aea8ecc 2024-11-13T05:55:17,848 INFO [StoreOpener-b540ac2e8c4575b424d481848c4b4e52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b540ac2e8c4575b424d481848c4b4e52 columnFamilyName colfamily21 2024-11-13T05:55:17,848 DEBUG [StoreOpener-b540ac2e8c4575b424d481848c4b4e52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,849 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=e86ecaa5f83edede0f153db264078a94, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:55:17,849 INFO [StoreOpener-89e8c9d6d5af67c75803894f4aea8ecc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 89e8c9d6d5af67c75803894f4aea8ecc 2024-11-13T05:55:17,849 INFO [StoreOpener-b540ac2e8c4575b424d481848c4b4e52-1 {}] regionserver.HStore(327): Store=b540ac2e8c4575b424d481848c4b4e52/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,850 INFO [StoreOpener-b540ac2e8c4575b424d481848c4b4e52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region b540ac2e8c4575b424d481848c4b4e52 2024-11-13T05:55:17,850 INFO [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98cb12bd0a37bbbfee08e3459a85f735 columnFamilyName colfamily31 2024-11-13T05:55:17,850 DEBUG [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,852 INFO [StoreOpener-89e8c9d6d5af67c75803894f4aea8ecc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89e8c9d6d5af67c75803894f4aea8ecc columnFamilyName colfamily11 2024-11-13T05:55:17,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure e86ecaa5f83edede0f153db264078a94, server=bc386b7bf2fe,40327,1731477314499 because future has completed 2024-11-13T05:55:17,852 DEBUG [StoreOpener-89e8c9d6d5af67c75803894f4aea8ecc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,852 INFO [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] regionserver.HStore(327): Store=98cb12bd0a37bbbfee08e3459a85f735/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,853 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:55:17,854 INFO [StoreOpener-b540ac2e8c4575b424d481848c4b4e52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b540ac2e8c4575b424d481848c4b4e52 columnFamilyName colfamily31 2024-11-13T05:55:17,854 DEBUG [StoreOpener-b540ac2e8c4575b424d481848c4b4e52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,854 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:55:17,854 INFO [StoreOpener-89e8c9d6d5af67c75803894f4aea8ecc-1 {}] regionserver.HStore(327): Store=89e8c9d6d5af67c75803894f4aea8ecc/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,855 INFO [StoreOpener-b540ac2e8c4575b424d481848c4b4e52-1 {}] regionserver.HStore(327): Store=b540ac2e8c4575b424d481848c4b4e52/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,855 INFO [StoreOpener-89e8c9d6d5af67c75803894f4aea8ecc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 89e8c9d6d5af67c75803894f4aea8ecc 2024-11-13T05:55:17,855 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for b540ac2e8c4575b424d481848c4b4e52 2024-11-13T05:55:17,855 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:55:17,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=19321613b69704e42207868d450f785b, ASSIGN in 619 msec 2024-11-13T05:55:17,856 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b540ac2e8c4575b424d481848c4b4e52 2024-11-13T05:55:17,857 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b540ac2e8c4575b424d481848c4b4e52 2024-11-13T05:55:17,857 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:55:17,857 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:55:17,858 INFO [StoreOpener-89e8c9d6d5af67c75803894f4aea8ecc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89e8c9d6d5af67c75803894f4aea8ecc columnFamilyName colfamily21 2024-11-13T05:55:17,858 DEBUG [StoreOpener-89e8c9d6d5af67c75803894f4aea8ecc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,858 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,859 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for b540ac2e8c4575b424d481848c4b4e52 2024-11-13T05:55:17,859 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for b540ac2e8c4575b424d481848c4b4e52 2024-11-13T05:55:17,859 INFO [StoreOpener-89e8c9d6d5af67c75803894f4aea8ecc-1 {}] regionserver.HStore(327): Store=89e8c9d6d5af67c75803894f4aea8ecc/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,860 INFO [StoreOpener-89e8c9d6d5af67c75803894f4aea8ecc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 89e8c9d6d5af67c75803894f4aea8ecc 2024-11-13T05:55:17,860 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,861 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:55:17,862 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for b540ac2e8c4575b424d481848c4b4e52 2024-11-13T05:55:17,863 INFO [StoreOpener-89e8c9d6d5af67c75803894f4aea8ecc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89e8c9d6d5af67c75803894f4aea8ecc columnFamilyName colfamily31 2024-11-13T05:55:17,863 DEBUG [StoreOpener-89e8c9d6d5af67c75803894f4aea8ecc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,864 INFO [StoreOpener-89e8c9d6d5af67c75803894f4aea8ecc-1 {}] regionserver.HStore(327): Store=89e8c9d6d5af67c75803894f4aea8ecc/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,864 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 89e8c9d6d5af67c75803894f4aea8ecc 2024-11-13T05:55:17,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=25 2024-11-13T05:55:17,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure e86ecaa5f83edede0f153db264078a94, server=bc386b7bf2fe,40327,1731477314499 in 382 msec 2024-11-13T05:55:17,866 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,866 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b540ac2e8c4575b424d481848c4b4e52/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,866 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/89e8c9d6d5af67c75803894f4aea8ecc 2024-11-13T05:55:17,867 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/89e8c9d6d5af67c75803894f4aea8ecc 2024-11-13T05:55:17,867 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 98cb12bd0a37bbbfee08e3459a85f735; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71457647, jitterRate=0.06480191648006439}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,867 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:55:17,867 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened b540ac2e8c4575b424d481848c4b4e52; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58869240, jitterRate=-0.1227799654006958}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,867 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 98cb12bd0a37bbbfee08e3459a85f735: Running coprocessor pre-open hook at 1731477317820Writing region info on filesystem at 1731477317820Initializing all the Stores at 1731477317821 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317822 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317838 (+16 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317838Cleaning up temporary data from old regions at 1731477317857 (+19 ms)Running coprocessor post-open hooks at 1731477317867 (+10 ms)Region opened successfully at 1731477317867 2024-11-13T05:55:17,867 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b540ac2e8c4575b424d481848c4b4e52 2024-11-13T05:55:17,867 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for b540ac2e8c4575b424d481848c4b4e52: Running coprocessor pre-open hook at 1731477317829Writing region info on filesystem at 1731477317829Initializing all the Stores at 1731477317832 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317832Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317839 (+7 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317839Cleaning up temporary data from old regions at 1731477317859 (+20 ms)Running coprocessor post-open hooks at 1731477317867 (+8 ms)Region opened successfully at 1731477317867 2024-11-13T05:55:17,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e86ecaa5f83edede0f153db264078a94, ASSIGN in 632 msec 2024-11-13T05:55:17,868 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52., pid=36, masterSystemTime=1731477317564 2024-11-13T05:55:17,869 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735., pid=55, masterSystemTime=1731477317681 2024-11-13T05:55:17,870 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=a65241dbe6c2863f31a0138a73130225, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:17,871 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 89e8c9d6d5af67c75803894f4aea8ecc 2024-11-13T05:55:17,871 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 89e8c9d6d5af67c75803894f4aea8ecc 2024-11-13T05:55:17,873 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52. 2024-11-13T05:55:17,873 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52. 2024-11-13T05:55:17,873 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b. 2024-11-13T05:55:17,874 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => ff62c0b5be01546e2b1563e065750a3b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-13T05:55:17,874 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,874 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true ff62c0b5be01546e2b1563e065750a3b 2024-11-13T05:55:17,874 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure a65241dbe6c2863f31a0138a73130225, server=bc386b7bf2fe,38823,1731477314332 because future has completed 2024-11-13T05:55:17,874 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for ff62c0b5be01546e2b1563e065750a3b 2024-11-13T05:55:17,874 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for ff62c0b5be01546e2b1563e065750a3b 2024-11-13T05:55:17,876 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735. 2024-11-13T05:55:17,876 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735. 2024-11-13T05:55:17,876 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac. 2024-11-13T05:55:17,876 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => e903b94793c361b461207b7aa3c133ac, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-13T05:55:17,876 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 89e8c9d6d5af67c75803894f4aea8ecc 2024-11-13T05:55:17,877 INFO [StoreOpener-ff62c0b5be01546e2b1563e065750a3b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region ff62c0b5be01546e2b1563e065750a3b 2024-11-13T05:55:17,877 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true e903b94793c361b461207b7aa3c133ac 2024-11-13T05:55:17,877 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,877 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for e903b94793c361b461207b7aa3c133ac 2024-11-13T05:55:17,877 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for e903b94793c361b461207b7aa3c133ac 2024-11-13T05:55:17,880 INFO [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region e903b94793c361b461207b7aa3c133ac 2024-11-13T05:55:17,880 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/89e8c9d6d5af67c75803894f4aea8ecc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,880 INFO [StoreOpener-ff62c0b5be01546e2b1563e065750a3b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff62c0b5be01546e2b1563e065750a3b columnFamilyName colfamily11 2024-11-13T05:55:17,881 DEBUG [StoreOpener-ff62c0b5be01546e2b1563e065750a3b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,881 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 89e8c9d6d5af67c75803894f4aea8ecc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61404488, jitterRate=-0.08500182628631592}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,881 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 89e8c9d6d5af67c75803894f4aea8ecc 2024-11-13T05:55:17,881 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 89e8c9d6d5af67c75803894f4aea8ecc: Running coprocessor pre-open hook at 1731477317847Writing region info on filesystem at 1731477317847Initializing all the Stores at 1731477317849 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317849Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317849Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317849Cleaning up temporary data from old regions at 1731477317871 (+22 ms)Running coprocessor post-open hooks at 1731477317881 (+10 ms)Region opened successfully at 1731477317881 2024-11-13T05:55:17,882 INFO [StoreOpener-ff62c0b5be01546e2b1563e065750a3b-1 {}] regionserver.HStore(327): Store=ff62c0b5be01546e2b1563e065750a3b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,882 INFO [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e903b94793c361b461207b7aa3c133ac columnFamilyName colfamily11 2024-11-13T05:55:17,882 INFO [StoreOpener-ff62c0b5be01546e2b1563e065750a3b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region ff62c0b5be01546e2b1563e065750a3b 2024-11-13T05:55:17,883 DEBUG [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,883 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc., pid=43, masterSystemTime=1731477317628 2024-11-13T05:55:17,884 INFO [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] regionserver.HStore(327): Store=e903b94793c361b461207b7aa3c133ac/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,884 INFO [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region e903b94793c361b461207b7aa3c133ac 2024-11-13T05:55:17,886 INFO [StoreOpener-ff62c0b5be01546e2b1563e065750a3b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff62c0b5be01546e2b1563e065750a3b columnFamilyName colfamily21 2024-11-13T05:55:17,886 DEBUG [StoreOpener-ff62c0b5be01546e2b1563e065750a3b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,886 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc. 2024-11-13T05:55:17,886 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc. 2024-11-13T05:55:17,887 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749. 2024-11-13T05:55:17,887 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 9bd21adcd80ce6ee8c5ed1a581c04749, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-13T05:55:17,887 INFO [StoreOpener-ff62c0b5be01546e2b1563e065750a3b-1 {}] regionserver.HStore(327): Store=ff62c0b5be01546e2b1563e065750a3b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,887 INFO [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e903b94793c361b461207b7aa3c133ac columnFamilyName colfamily21 2024-11-13T05:55:17,887 DEBUG [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,887 INFO [StoreOpener-ff62c0b5be01546e2b1563e065750a3b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region ff62c0b5be01546e2b1563e065750a3b 2024-11-13T05:55:17,887 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 9bd21adcd80ce6ee8c5ed1a581c04749 2024-11-13T05:55:17,887 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,887 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 9bd21adcd80ce6ee8c5ed1a581c04749 2024-11-13T05:55:17,888 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 9bd21adcd80ce6ee8c5ed1a581c04749 2024-11-13T05:55:17,888 INFO [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] regionserver.HStore(327): Store=e903b94793c361b461207b7aa3c133ac/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,888 INFO [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region e903b94793c361b461207b7aa3c133ac 2024-11-13T05:55:17,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=26 2024-11-13T05:55:17,889 INFO [StoreOpener-ff62c0b5be01546e2b1563e065750a3b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff62c0b5be01546e2b1563e065750a3b columnFamilyName colfamily31 2024-11-13T05:55:17,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure a65241dbe6c2863f31a0138a73130225, server=bc386b7bf2fe,38823,1731477314332 in 318 msec 2024-11-13T05:55:17,889 DEBUG [StoreOpener-ff62c0b5be01546e2b1563e065750a3b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,890 INFO [StoreOpener-ff62c0b5be01546e2b1563e065750a3b-1 {}] regionserver.HStore(327): Store=ff62c0b5be01546e2b1563e065750a3b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,890 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for ff62c0b5be01546e2b1563e065750a3b 2024-11-13T05:55:17,891 INFO [StoreOpener-9bd21adcd80ce6ee8c5ed1a581c04749-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 9bd21adcd80ce6ee8c5ed1a581c04749 2024-11-13T05:55:17,891 INFO [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e903b94793c361b461207b7aa3c133ac columnFamilyName colfamily31 2024-11-13T05:55:17,891 DEBUG [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,892 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ff62c0b5be01546e2b1563e065750a3b 2024-11-13T05:55:17,892 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ff62c0b5be01546e2b1563e065750a3b 2024-11-13T05:55:17,893 INFO [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] regionserver.HStore(327): Store=e903b94793c361b461207b7aa3c133ac/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,893 INFO [StoreOpener-9bd21adcd80ce6ee8c5ed1a581c04749-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bd21adcd80ce6ee8c5ed1a581c04749 columnFamilyName colfamily11 2024-11-13T05:55:17,893 DEBUG [StoreOpener-9bd21adcd80ce6ee8c5ed1a581c04749-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,893 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for e903b94793c361b461207b7aa3c133ac 2024-11-13T05:55:17,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a65241dbe6c2863f31a0138a73130225, ASSIGN in 656 msec 2024-11-13T05:55:17,894 INFO [StoreOpener-9bd21adcd80ce6ee8c5ed1a581c04749-1 {}] regionserver.HStore(327): Store=9bd21adcd80ce6ee8c5ed1a581c04749/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,894 INFO [StoreOpener-9bd21adcd80ce6ee8c5ed1a581c04749-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 9bd21adcd80ce6ee8c5ed1a581c04749 2024-11-13T05:55:17,895 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac 2024-11-13T05:55:17,895 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for ff62c0b5be01546e2b1563e065750a3b 2024-11-13T05:55:17,895 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for ff62c0b5be01546e2b1563e065750a3b 2024-11-13T05:55:17,895 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac 2024-11-13T05:55:17,895 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=89e8c9d6d5af67c75803894f4aea8ecc, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:55:17,896 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,896 INFO [StoreOpener-9bd21adcd80ce6ee8c5ed1a581c04749-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bd21adcd80ce6ee8c5ed1a581c04749 columnFamilyName colfamily21 2024-11-13T05:55:17,896 DEBUG [StoreOpener-9bd21adcd80ce6ee8c5ed1a581c04749-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,897 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for e903b94793c361b461207b7aa3c133ac 2024-11-13T05:55:17,897 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for e903b94793c361b461207b7aa3c133ac 2024-11-13T05:55:17,897 INFO [StoreOpener-9bd21adcd80ce6ee8c5ed1a581c04749-1 {}] regionserver.HStore(327): Store=9bd21adcd80ce6ee8c5ed1a581c04749/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,897 INFO [StoreOpener-9bd21adcd80ce6ee8c5ed1a581c04749-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 9bd21adcd80ce6ee8c5ed1a581c04749 2024-11-13T05:55:17,898 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,900 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for ff62c0b5be01546e2b1563e065750a3b 2024-11-13T05:55:17,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 89e8c9d6d5af67c75803894f4aea8ecc, server=bc386b7bf2fe,40327,1731477314499 because future has completed 2024-11-13T05:55:17,901 INFO [StoreOpener-9bd21adcd80ce6ee8c5ed1a581c04749-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bd21adcd80ce6ee8c5ed1a581c04749 columnFamilyName colfamily31 2024-11-13T05:55:17,901 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for e903b94793c361b461207b7aa3c133ac 2024-11-13T05:55:17,901 DEBUG [StoreOpener-9bd21adcd80ce6ee8c5ed1a581c04749-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,902 INFO [StoreOpener-9bd21adcd80ce6ee8c5ed1a581c04749-1 {}] regionserver.HStore(327): Store=9bd21adcd80ce6ee8c5ed1a581c04749/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,902 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=98cb12bd0a37bbbfee08e3459a85f735, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:17,902 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 9bd21adcd80ce6ee8c5ed1a581c04749 2024-11-13T05:55:17,904 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9bd21adcd80ce6ee8c5ed1a581c04749 2024-11-13T05:55:17,905 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9bd21adcd80ce6ee8c5ed1a581c04749 2024-11-13T05:55:17,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 98cb12bd0a37bbbfee08e3459a85f735, server=bc386b7bf2fe,38823,1731477314332 because future has completed 2024-11-13T05:55:17,906 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 9bd21adcd80ce6ee8c5ed1a581c04749 2024-11-13T05:55:17,906 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 9bd21adcd80ce6ee8c5ed1a581c04749 2024-11-13T05:55:17,907 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,908 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ff62c0b5be01546e2b1563e065750a3b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,908 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,908 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened ff62c0b5be01546e2b1563e065750a3b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66134368, jitterRate=-0.014521121978759766}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,909 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ff62c0b5be01546e2b1563e065750a3b 2024-11-13T05:55:17,909 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for ff62c0b5be01546e2b1563e065750a3b: Running coprocessor pre-open hook at 1731477317874Writing region info on filesystem at 1731477317874Initializing all the Stores at 1731477317876 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317876Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317876Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317876Cleaning up temporary data from old regions at 1731477317895 (+19 ms)Running coprocessor post-open hooks at 1731477317909 (+14 ms)Region opened successfully at 1731477317909 2024-11-13T05:55:17,909 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened e903b94793c361b461207b7aa3c133ac; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71415129, jitterRate=0.06416834890842438}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,909 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e903b94793c361b461207b7aa3c133ac 2024-11-13T05:55:17,909 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for e903b94793c361b461207b7aa3c133ac: Running coprocessor pre-open hook at 1731477317877Writing region info on filesystem at 1731477317877Initializing all the Stores at 1731477317879 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317879Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317879Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317879Cleaning up temporary data from old regions at 1731477317897 (+18 ms)Running coprocessor post-open hooks at 1731477317909 (+12 ms)Region opened successfully at 1731477317909 2024-11-13T05:55:17,912 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b., pid=39, masterSystemTime=1731477317564 2024-11-13T05:55:17,912 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac., pid=50, masterSystemTime=1731477317681 2024-11-13T05:55:17,913 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 9bd21adcd80ce6ee8c5ed1a581c04749 2024-11-13T05:55:17,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=13 2024-11-13T05:55:17,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 89e8c9d6d5af67c75803894f4aea8ecc, server=bc386b7bf2fe,40327,1731477314499 in 420 msec 2024-11-13T05:55:17,917 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9bd21adcd80ce6ee8c5ed1a581c04749/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,918 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 9bd21adcd80ce6ee8c5ed1a581c04749; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59559269, jitterRate=-0.1124977320432663}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,918 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9bd21adcd80ce6ee8c5ed1a581c04749 2024-11-13T05:55:17,918 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 9bd21adcd80ce6ee8c5ed1a581c04749: Running coprocessor pre-open hook at 1731477317888Writing region info on filesystem at 1731477317888Initializing all the Stores at 1731477317889 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317889Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317890 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317890Cleaning up temporary data from old regions at 1731477317906 (+16 ms)Running coprocessor post-open hooks at 1731477317918 (+12 ms)Region opened successfully at 1731477317918 2024-11-13T05:55:17,920 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b. 2024-11-13T05:55:17,918 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac. 2024-11-13T05:55:17,920 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b. 2024-11-13T05:55:17,920 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac. 2024-11-13T05:55:17,920 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1. 2024-11-13T05:55:17,920 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749., pid=45, masterSystemTime=1731477317628 2024-11-13T05:55:17,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=89e8c9d6d5af67c75803894f4aea8ecc, ASSIGN in 683 msec 2024-11-13T05:55:17,920 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 11abc6d6caf592b351052ae8adf50fd1, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-13T05:55:17,921 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:55:17,921 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,921 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:55:17,921 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:55:17,922 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=ff62c0b5be01546e2b1563e065750a3b, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:17,923 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749. 2024-11-13T05:55:17,923 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749. 2024-11-13T05:55:17,923 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b. 2024-11-13T05:55:17,923 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a492bcf61cc6ef0604137033fc7c90b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-13T05:55:17,924 INFO [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:55:17,925 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 1a492bcf61cc6ef0604137033fc7c90b 2024-11-13T05:55:17,925 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,925 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 1a492bcf61cc6ef0604137033fc7c90b 2024-11-13T05:55:17,925 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 1a492bcf61cc6ef0604137033fc7c90b 2024-11-13T05:55:17,925 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff62c0b5be01546e2b1563e065750a3b, server=bc386b7bf2fe,45383,1731477314448 because future has completed 2024-11-13T05:55:17,927 INFO [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11abc6d6caf592b351052ae8adf50fd1 columnFamilyName colfamily11 2024-11-13T05:55:17,927 DEBUG [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,927 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=9bd21adcd80ce6ee8c5ed1a581c04749, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:55:17,927 INFO [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] regionserver.HStore(327): Store=11abc6d6caf592b351052ae8adf50fd1/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,928 INFO [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:55:17,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9bd21adcd80ce6ee8c5ed1a581c04749, server=bc386b7bf2fe,40327,1731477314499 because future has completed 2024-11-13T05:55:17,930 INFO [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11abc6d6caf592b351052ae8adf50fd1 columnFamilyName colfamily21 2024-11-13T05:55:17,930 DEBUG [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,931 INFO [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] regionserver.HStore(327): Store=11abc6d6caf592b351052ae8adf50fd1/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,931 INFO [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:55:17,933 INFO [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11abc6d6caf592b351052ae8adf50fd1 columnFamilyName colfamily31 2024-11-13T05:55:17,934 DEBUG [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,934 INFO [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] regionserver.HStore(327): Store=11abc6d6caf592b351052ae8adf50fd1/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=27 2024-11-13T05:55:17,935 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:55:17,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure ff62c0b5be01546e2b1563e065750a3b, server=bc386b7bf2fe,45383,1731477314448 in 468 msec 2024-11-13T05:55:17,936 INFO [StoreOpener-1a492bcf61cc6ef0604137033fc7c90b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 1a492bcf61cc6ef0604137033fc7c90b 2024-11-13T05:55:17,936 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:55:17,936 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:55:17,938 INFO [StoreOpener-1a492bcf61cc6ef0604137033fc7c90b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a492bcf61cc6ef0604137033fc7c90b columnFamilyName colfamily11 2024-11-13T05:55:17,938 DEBUG [StoreOpener-1a492bcf61cc6ef0604137033fc7c90b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,938 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:55:17,938 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:55:17,938 INFO [StoreOpener-1a492bcf61cc6ef0604137033fc7c90b-1 {}] regionserver.HStore(327): Store=1a492bcf61cc6ef0604137033fc7c90b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,939 INFO [StoreOpener-1a492bcf61cc6ef0604137033fc7c90b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 1a492bcf61cc6ef0604137033fc7c90b 2024-11-13T05:55:17,939 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,940 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=bc386b7bf2fe,38823,1731477314332, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9afb90ba9af49bb3547d4438986e1fca. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T05:55:17,940 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=bc386b7bf2fe,38823,1731477314332, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=98cb12bd0a37bbbfee08e3459a85f735. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T05:55:17,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ff62c0b5be01546e2b1563e065750a3b, ASSIGN in 702 msec 2024-11-13T05:55:17,942 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:55:17,943 INFO [StoreOpener-1a492bcf61cc6ef0604137033fc7c90b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a492bcf61cc6ef0604137033fc7c90b columnFamilyName colfamily21 2024-11-13T05:55:17,943 DEBUG [StoreOpener-1a492bcf61cc6ef0604137033fc7c90b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,944 INFO [StoreOpener-1a492bcf61cc6ef0604137033fc7c90b-1 {}] regionserver.HStore(327): Store=1a492bcf61cc6ef0604137033fc7c90b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,944 INFO [StoreOpener-1a492bcf61cc6ef0604137033fc7c90b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 1a492bcf61cc6ef0604137033fc7c90b 2024-11-13T05:55:17,946 INFO [StoreOpener-1a492bcf61cc6ef0604137033fc7c90b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a492bcf61cc6ef0604137033fc7c90b columnFamilyName colfamily31 2024-11-13T05:55:17,946 DEBUG [StoreOpener-1a492bcf61cc6ef0604137033fc7c90b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,947 INFO [StoreOpener-1a492bcf61cc6ef0604137033fc7c90b-1 {}] regionserver.HStore(327): Store=1a492bcf61cc6ef0604137033fc7c90b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,947 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,947 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 1a492bcf61cc6ef0604137033fc7c90b 2024-11-13T05:55:17,948 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1a492bcf61cc6ef0604137033fc7c90b 2024-11-13T05:55:17,948 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 11abc6d6caf592b351052ae8adf50fd1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61427411, jitterRate=-0.08466024696826935}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,948 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:55:17,948 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 11abc6d6caf592b351052ae8adf50fd1: Running coprocessor pre-open hook at 1731477317921Writing region info on filesystem at 1731477317921Initializing all the Stores at 1731477317923 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317923Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317923Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317923Cleaning up temporary data from old regions at 1731477317938 (+15 ms)Running coprocessor post-open hooks at 1731477317948 (+10 ms)Region opened successfully at 1731477317948 2024-11-13T05:55:17,948 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1a492bcf61cc6ef0604137033fc7c90b 2024-11-13T05:55:17,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=22 2024-11-13T05:55:17,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 9bd21adcd80ce6ee8c5ed1a581c04749, server=bc386b7bf2fe,40327,1731477314499 in 444 msec 2024-11-13T05:55:17,950 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1., pid=51, masterSystemTime=1731477317681 2024-11-13T05:55:17,950 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 1a492bcf61cc6ef0604137033fc7c90b 2024-11-13T05:55:17,950 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 1a492bcf61cc6ef0604137033fc7c90b 2024-11-13T05:55:17,951 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9bd21adcd80ce6ee8c5ed1a581c04749, ASSIGN in 716 msec 2024-11-13T05:55:17,953 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1. 2024-11-13T05:55:17,953 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1. 2024-11-13T05:55:17,953 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7. 2024-11-13T05:55:17,953 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 82c0a84200b3abe2cb346258143147c7, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-13T05:55:17,954 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:55:17,954 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,954 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:55:17,954 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:55:17,954 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 1a492bcf61cc6ef0604137033fc7c90b 2024-11-13T05:55:17,955 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=11abc6d6caf592b351052ae8adf50fd1, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:17,957 INFO [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:55:17,959 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1a492bcf61cc6ef0604137033fc7c90b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11abc6d6caf592b351052ae8adf50fd1, server=bc386b7bf2fe,38823,1731477314332 because future has completed 2024-11-13T05:55:17,960 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 1a492bcf61cc6ef0604137033fc7c90b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75156233, jitterRate=0.11991514265537262}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,961 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a492bcf61cc6ef0604137033fc7c90b 2024-11-13T05:55:17,961 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=e903b94793c361b461207b7aa3c133ac, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:17,961 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 1a492bcf61cc6ef0604137033fc7c90b: Running coprocessor pre-open hook at 1731477317925Writing region info on filesystem at 1731477317925Initializing all the Stores at 1731477317927 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317927Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317934 (+7 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317934Cleaning up temporary data from old regions at 1731477317950 (+16 ms)Running coprocessor post-open hooks at 1731477317961 (+11 ms)Region opened successfully at 1731477317961 2024-11-13T05:55:17,961 INFO [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82c0a84200b3abe2cb346258143147c7 columnFamilyName colfamily11 2024-11-13T05:55:17,961 DEBUG [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,962 INFO [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] regionserver.HStore(327): Store=82c0a84200b3abe2cb346258143147c7/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,962 INFO [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:55:17,964 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b., pid=46, masterSystemTime=1731477317628 2024-11-13T05:55:17,965 INFO [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82c0a84200b3abe2cb346258143147c7 columnFamilyName colfamily21 2024-11-13T05:55:17,965 DEBUG [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,966 INFO [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] regionserver.HStore(327): Store=82c0a84200b3abe2cb346258143147c7/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,966 INFO [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:55:17,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure e903b94793c361b461207b7aa3c133ac, server=bc386b7bf2fe,38823,1731477314332 because future has completed 2024-11-13T05:55:17,968 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b. 2024-11-13T05:55:17,968 INFO [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82c0a84200b3abe2cb346258143147c7 columnFamilyName colfamily31 2024-11-13T05:55:17,968 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b. 2024-11-13T05:55:17,968 DEBUG [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,969 INFO [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] regionserver.HStore(327): Store=82c0a84200b3abe2cb346258143147c7/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,969 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:55:17,970 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7 2024-11-13T05:55:17,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=11 2024-11-13T05:55:17,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 11abc6d6caf592b351052ae8adf50fd1, server=bc386b7bf2fe,38823,1731477314332 in 417 msec 2024-11-13T05:55:17,971 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7 2024-11-13T05:55:17,972 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:55:17,972 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:55:17,973 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:17,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=11abc6d6caf592b351052ae8adf50fd1, ASSIGN in 738 msec 2024-11-13T05:55:17,974 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:55:17,977 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1a492bcf61cc6ef0604137033fc7c90b, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:55:17,977 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:17,978 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 82c0a84200b3abe2cb346258143147c7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67840575, jitterRate=0.010903343558311462}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:17,978 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:55:17,978 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 82c0a84200b3abe2cb346258143147c7: Running coprocessor pre-open hook at 1731477317954Writing region info on filesystem at 1731477317954Initializing all the Stores at 1731477317955 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317955Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317957 (+2 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317957Cleaning up temporary data from old regions at 1731477317972 (+15 ms)Running coprocessor post-open hooks at 1731477317978 (+6 ms)Region opened successfully at 1731477317978 2024-11-13T05:55:17,980 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7., pid=47, masterSystemTime=1731477317681 2024-11-13T05:55:17,981 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a492bcf61cc6ef0604137033fc7c90b, server=bc386b7bf2fe,40327,1731477314499 because future has completed 2024-11-13T05:55:17,982 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7. 2024-11-13T05:55:17,983 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7. 2024-11-13T05:55:17,983 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d. 2024-11-13T05:55:17,983 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 84316f2c5b758522338a907d7e8f386d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-13T05:55:17,983 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:55:17,983 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:17,984 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:55:17,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=8 2024-11-13T05:55:17,984 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:55:17,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure e903b94793c361b461207b7aa3c133ac, server=bc386b7bf2fe,38823,1731477314332 in 438 msec 2024-11-13T05:55:17,986 INFO [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:55:17,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e903b94793c361b461207b7aa3c133ac, ASSIGN in 751 msec 2024-11-13T05:55:17,988 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=82c0a84200b3abe2cb346258143147c7, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:17,988 INFO [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 84316f2c5b758522338a907d7e8f386d columnFamilyName colfamily11 2024-11-13T05:55:17,989 DEBUG [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,989 INFO [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] regionserver.HStore(327): Store=84316f2c5b758522338a907d7e8f386d/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,990 INFO [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:55:17,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82c0a84200b3abe2cb346258143147c7, server=bc386b7bf2fe,38823,1731477314332 because future has completed 2024-11-13T05:55:17,992 INFO [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 84316f2c5b758522338a907d7e8f386d columnFamilyName colfamily21 2024-11-13T05:55:17,993 DEBUG [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,993 INFO [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] regionserver.HStore(327): Store=84316f2c5b758522338a907d7e8f386d/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,993 INFO [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:55:17,995 INFO [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 84316f2c5b758522338a907d7e8f386d columnFamilyName colfamily31 2024-11-13T05:55:17,996 DEBUG [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:17,996 INFO [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] regionserver.HStore(327): Store=84316f2c5b758522338a907d7e8f386d/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:17,997 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:55:17,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=10 2024-11-13T05:55:17,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a492bcf61cc6ef0604137033fc7c90b, server=bc386b7bf2fe,40327,1731477314499 in 475 msec 2024-11-13T05:55:17,998 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d 2024-11-13T05:55:17,999 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d 2024-11-13T05:55:18,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1a492bcf61cc6ef0604137033fc7c90b, ASSIGN in 764 msec 2024-11-13T05:55:18,001 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:55:18,001 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:55:18,002 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:18,005 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:55:18,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=23 2024-11-13T05:55:18,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 82c0a84200b3abe2cb346258143147c7, server=bc386b7bf2fe,38823,1731477314332 in 476 msec 2024-11-13T05:55:18,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=82c0a84200b3abe2cb346258143147c7, ASSIGN in 772 msec 2024-11-13T05:55:18,009 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:18,010 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 84316f2c5b758522338a907d7e8f386d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63755696, jitterRate=-0.04996609687805176}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:18,010 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:55:18,010 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 84316f2c5b758522338a907d7e8f386d: Running coprocessor pre-open hook at 1731477317984Writing region info on filesystem at 1731477317984Initializing all the Stores at 1731477317986 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317986Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317986Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477317986Cleaning up temporary data from old regions at 1731477318002 (+16 ms)Running coprocessor post-open hooks at 1731477318010 (+8 ms)Region opened successfully at 1731477318010 2024-11-13T05:55:18,011 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d., pid=49, masterSystemTime=1731477317681 2024-11-13T05:55:18,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=29 2024-11-13T05:55:18,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 98cb12bd0a37bbbfee08e3459a85f735, server=bc386b7bf2fe,38823,1731477314332 in 444 msec 2024-11-13T05:55:18,014 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d. 2024-11-13T05:55:18,014 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d. 2024-11-13T05:55:18,014 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222. 2024-11-13T05:55:18,014 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b613e7fdd93a34284c913bfa5a7b222, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-13T05:55:18,015 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:55:18,015 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:55:18,015 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:55:18,015 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:55:18,018 INFO [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:55:18,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=98cb12bd0a37bbbfee08e3459a85f735, ASSIGN in 780 msec 2024-11-13T05:55:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:55:18,020 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=84316f2c5b758522338a907d7e8f386d, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:18,021 INFO [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b613e7fdd93a34284c913bfa5a7b222 columnFamilyName colfamily11 2024-11-13T05:55:18,021 DEBUG [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:18,021 INFO [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] regionserver.HStore(327): Store=4b613e7fdd93a34284c913bfa5a7b222/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:18,021 INFO [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:55:18,024 INFO [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b613e7fdd93a34284c913bfa5a7b222 columnFamilyName colfamily21 2024-11-13T05:55:18,024 DEBUG [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:18,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 84316f2c5b758522338a907d7e8f386d, server=bc386b7bf2fe,38823,1731477314332 because future has completed 2024-11-13T05:55:18,025 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=b540ac2e8c4575b424d481848c4b4e52, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:55:18,025 INFO [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] regionserver.HStore(327): Store=4b613e7fdd93a34284c913bfa5a7b222/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:18,025 INFO [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:55:18,027 INFO [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b613e7fdd93a34284c913bfa5a7b222 columnFamilyName colfamily31 2024-11-13T05:55:18,028 DEBUG [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:55:18,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure b540ac2e8c4575b424d481848c4b4e52, server=bc386b7bf2fe,45383,1731477314448 because future has completed 2024-11-13T05:55:18,028 INFO [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] regionserver.HStore(327): Store=4b613e7fdd93a34284c913bfa5a7b222/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:55:18,028 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:55:18,029 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:55:18,030 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:55:18,031 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:55:18,031 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:55:18,032 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:55:18,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=5 2024-11-13T05:55:18,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 84316f2c5b758522338a907d7e8f386d, server=bc386b7bf2fe,38823,1731477314332 in 490 msec 2024-11-13T05:55:18,033 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:55:18,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=84316f2c5b758522338a907d7e8f386d, ASSIGN in 799 msec 2024-11-13T05:55:18,036 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4b613e7fdd93a34284c913bfa5a7b222/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:55:18,037 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 4b613e7fdd93a34284c913bfa5a7b222; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58825332, jitterRate=-0.12343424558639526}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:55:18,038 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:55:18,038 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 4b613e7fdd93a34284c913bfa5a7b222: Running coprocessor pre-open hook at 1731477318015Writing region info on filesystem at 1731477318015Initializing all the Stores at 1731477318017 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477318018 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477318018Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477318018Cleaning up temporary data from old regions at 1731477318031 (+13 ms)Running coprocessor post-open hooks at 1731477318038 (+7 ms)Region opened successfully at 1731477318038 2024-11-13T05:55:18,039 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222., pid=48, masterSystemTime=1731477317681 2024-11-13T05:55:18,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=15 2024-11-13T05:55:18,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure b540ac2e8c4575b424d481848c4b4e52, server=bc386b7bf2fe,45383,1731477314448 in 588 msec 2024-11-13T05:55:18,044 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222. 2024-11-13T05:55:18,044 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222. 2024-11-13T05:55:18,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b540ac2e8c4575b424d481848c4b4e52, ASSIGN in 807 msec 2024-11-13T05:55:18,046 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=bc386b7bf2fe,38823,1731477314332, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9afb90ba9af49bb3547d4438986e1fca. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T05:55:18,046 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=4b613e7fdd93a34284c913bfa5a7b222, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:18,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b613e7fdd93a34284c913bfa5a7b222, server=bc386b7bf2fe,38823,1731477314332 because future has completed 2024-11-13T05:55:18,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=20 2024-11-13T05:55:18,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 9afb90ba9af49bb3547d4438986e1fca, server=bc386b7bf2fe,38823,1731477314332 in 492 msec 2024-11-13T05:55:18,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9afb90ba9af49bb3547d4438986e1fca, ASSIGN in 820 msec 2024-11-13T05:55:18,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=14 2024-11-13T05:55:18,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 4b613e7fdd93a34284c913bfa5a7b222, server=bc386b7bf2fe,38823,1731477314332 in 523 msec 2024-11-13T05:55:18,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=4 2024-11-13T05:55:18,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4b613e7fdd93a34284c913bfa5a7b222, ASSIGN in 827 msec 2024-11-13T05:55:18,066 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T05:55:18,066 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477318066"}]},"ts":"1731477318066"} 2024-11-13T05:55:18,069 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, state=ENABLED in hbase:meta 2024-11-13T05:55:18,071 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T05:55:18,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true in 1.2090 sec 2024-11-13T05:55:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:55:19,031 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true completed 2024-11-13T05:55:19,031 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true get assigned. Timeout = 60000ms 2024-11-13T05:55:19,033 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T05:55:19,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38823 {}] regionserver.StoreScanner(1138): Switch to stream read (scanned=32802 bytes) of info 2024-11-13T05:55:19,057 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true assigned to meta. Checking AM states. 2024-11-13T05:55:19,057 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T05:55:19,058 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true assigned. 2024-11-13T05:55:19,117 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aab', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,120 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aaa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,122 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aac', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,123 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aad', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,124 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aae', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,125 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aaf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,126 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aag', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,127 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aah', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,172 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,175 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,176 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,178 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,179 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbe', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,220 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,222 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,224 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='cce', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,226 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,228 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,229 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='cch', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,266 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='dde', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,268 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,270 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,272 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,274 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,276 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,277 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,315 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eee', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,317 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eef', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,319 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eeg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,321 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eeh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,322 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eei', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,323 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eej', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,366 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,368 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='fff', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,370 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,372 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,373 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,374 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,415 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,417 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,419 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,422 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,424 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,426 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,462 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,464 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,465 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,467 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,473 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,475 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,507 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iii', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,509 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iij', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,511 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iik', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,513 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iil', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,514 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iin', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,516 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iim', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,552 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,554 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,556 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,558 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,559 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,610 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,614 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,616 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,618 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,621 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kko', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,623 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,656 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lll', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,658 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,660 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,663 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lln', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,665 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,667 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,669 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,704 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,706 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,709 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,711 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,713 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,715 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,748 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,750 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nno', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,752 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,754 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,758 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,760 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nns', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,789 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ooo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,791 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oop', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,793 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ooq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,795 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oor', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,798 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oos', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,800 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oot', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,835 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,838 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,840 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,842 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='pps', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,844 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,875 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,877 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,879 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqs', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,880 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,882 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:19,944 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,945 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrs', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,947 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,949 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rru', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:19,997 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sst', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:19,999 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sss', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:20,001 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:20,002 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:20,004 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:20,006 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:20,037 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:20,039 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:20,041 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:20,043 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:20,045 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:20,082 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:20,084 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:20,086 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uux', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:20,088 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:20,089 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:20,091 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:20,151 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:20,153 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:20,155 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:20,157 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:20,158 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vwa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:20,160 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:20,199 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:20,201 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='www', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:20,202 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:20,204 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:20,205 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wxa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:20,257 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:20,259 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:20,261 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:20,263 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xya', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:20,265 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xyb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=2] 2024-11-13T05:55:20,299 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:20,301 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:20,303 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:20,304 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yza', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:20,306 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:20,307 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2] 2024-11-13T05:55:20,356 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='zzz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=2] 2024-11-13T05:55:20,376 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:55:20,390 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:55:20,391 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:55:20,404 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:55:20,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45383 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 5625 Client: jenkins//172.17.0.2 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6. 2024-11-13T05:55:20,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38823 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 6327 Client: jenkins//172.17.0.2 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d. 2024-11-13T05:55:20,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40327 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 5624 Client: jenkins//172.17.0.2 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e. 2024-11-13T05:55:21,257 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,, stopping at row=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true ,, for max=2147483647 with caching=100 2024-11-13T05:55:21,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38823 {}] regionserver.StoreScanner(1138): Switch to stream read (scanned=32802 bytes) of info 2024-11-13T05:55:21,519 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: true 2024-11-13T05:55:21,519 INFO [Time-limited test {}] assignment.AssignmentTestingUtil(82): KILL REGION SERVER bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:21,519 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:21,520 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server bc386b7bf2fe,38823,1731477314332: Simulated kill ***** 2024-11-13T05:55:21,520 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-13T05:55:21,522 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-13T05:55:21,524 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-13T05:55:21,524 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-13T05:55:21,524 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-13T05:55:21,524 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 605425952 }, "NonHeapMemoryUsage": { "committed": 157417472, "init": 7667712, "max": -1, "used": 154528840 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [], "beans": [], "beans": [] } 2024-11-13T05:55:21,527 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37807 {}] master.MasterRpcServices(700): bc386b7bf2fe,38823,1731477314332 reported a fatal error: ***** ABORTING region server bc386b7bf2fe,38823,1731477314332: Simulated kill ***** 2024-11-13T05:55:21,529 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bc386b7bf2fe,38823,1731477314332' ***** 2024-11-13T05:55:21,529 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-13T05:55:21,529 INFO [RS:0;bc386b7bf2fe:38823 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T05:55:21,529 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T05:55:21,529 INFO [RS:0;bc386b7bf2fe:38823 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-13T05:55:21,529 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T05:55:21,530 INFO [RS:0;bc386b7bf2fe:38823 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-13T05:55:21,530 INFO [RS:0;bc386b7bf2fe:38823 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:55:21,530 INFO [RS:0;bc386b7bf2fe:38823 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;bc386b7bf2fe:38823. 2024-11-13T05:55:21,530 DEBUG [RS:0;bc386b7bf2fe:38823 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:55:21,537 DEBUG [RS:0;bc386b7bf2fe:38823 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:55:21,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741836_1012 (size=54214) 2024-11-13T05:55:21,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741836_1012 (size=54214) 2024-11-13T05:55:21,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741836_1012 (size=54214) 2024-11-13T05:55:21,544 INFO [regionserver/bc386b7bf2fe:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:55:21,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741834_1010 (size=334915) 2024-11-13T05:55:21,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741834_1010 (size=334915) 2024-11-13T05:55:21,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741834_1010 (size=334915) 2024-11-13T05:55:21,548 DEBUG [RS:0;bc386b7bf2fe:38823 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:55:21,548 INFO [RS:0;bc386b7bf2fe:38823 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:55:21,548 INFO [RS:0;bc386b7bf2fe:38823 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38823 2024-11-13T05:55:21,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38823-0x101340ca7aa0001, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bc386b7bf2fe,38823,1731477314332 2024-11-13T05:55:21,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:55:21,605 INFO [RS:0;bc386b7bf2fe:38823 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:55:21,607 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bc386b7bf2fe,38823,1731477314332] 2024-11-13T05:55:21,621 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bc386b7bf2fe,38823,1731477314332 already deleted, retry=false 2024-11-13T05:55:21,622 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of bc386b7bf2fe,38823,1731477314332 on bc386b7bf2fe,37807,1731477313603 2024-11-13T05:55:21,627 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:55:21,632 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:55:21,633 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for bc386b7bf2fe,38823,1731477314332 (carryingMeta=true) bc386b7bf2fe,38823,1731477314332/CRASHED/regionCount=10/lock=java.util.concurrent.locks.ReentrantReadWriteLock@5f95c74d[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-13T05:55:21,633 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T05:55:21,633 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:55:21,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:55:21,633 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:55:21,637 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T05:55:21,639 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 0 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:55:21,640 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T05:55:21,640 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:55:21,640 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T05:55:21,640 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T05:55:21,641 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T05:55:21,641 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T05:55:21,641 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T05:55:21,641 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T05:55:21,642 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T05:55:21,646 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T05:55:21,647 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T05:55:21,648 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T05:55:21,648 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:55:21,648 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T05:55:21,649 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-13T05:55:21,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:55:21,651 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bc386b7bf2fe,38823,1731477314332, table=hbase:meta, region=1588230740 2024-11-13T05:55:21,651 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,38823,1731477314332, state=OPEN 2024-11-13T05:55:21,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:55:21,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:55:21,655 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:55:21,655 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:55:21,655 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:55:21,655 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:55:21,657 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731477351657 2024-11-13T05:55:21,658 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T05:55:21,658 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:55:21,658 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T05:55:21,659 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:55:21,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:55:21,663 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:55:21,663 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:55:21,664 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:55:21,665 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true, isMeta: true 2024-11-13T05:55:21,667 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting 2024-11-13T05:55:21,669 INFO [PEWorker-1 {}] master.SplitWALManager(105): bc386b7bf2fe,38823,1731477314332 WAL count=1, meta=true 2024-11-13T05:55:21,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta}] 2024-11-13T05:55:21,672 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T05:55:21,672 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:55:21,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:55:21,672 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:55:21,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38823-0x101340ca7aa0001, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:55:21,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38823-0x101340ca7aa0001, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:55:21,714 INFO [RS:0;bc386b7bf2fe:38823 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:55:21,715 INFO [RS:0;bc386b7bf2fe:38823 {}] regionserver.HRegionServer(1031): Exiting; stopping=bc386b7bf2fe,38823,1731477314332; zookeeper connection closed. 2024-11-13T05:55:21,715 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5d6fd38e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5d6fd38e 2024-11-13T05:55:21,767 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:55:21,768 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:55:21,768 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:38823 this server is in the failed servers list 2024-11-13T05:55:21,769 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:55:21,769 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:55:21,769 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:55:21,921 INFO [regionserver/bc386b7bf2fe:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T05:55:21,921 INFO [regionserver/bc386b7bf2fe:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T05:55:21,976 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T05:55:21,978 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:55:21,978 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:55:21,979 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:38823 this server is in the failed servers list 2024-11-13T05:55:21,980 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:55:21,980 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:55:21,981 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:55:22,029 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T05:55:22,031 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true' 2024-11-13T05:55:22,287 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:55:22,288 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:55:22,288 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:38823 this server is in the failed servers list 2024-11-13T05:55:22,289 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:55:22,289 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:55:22,290 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:55:22,797 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:55:22,798 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:55:22,798 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:38823 this server is in the failed servers list 2024-11-13T05:55:22,798 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:55:22,798 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:55:22,798 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:55:23,817 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:55:23,817 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:55:23,819 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:55:23,825 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38823,1731477314332 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2162 ms java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T05:55:23,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:55:23,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:55:23,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:55:23,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:55:23,937 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T05:55:23,937 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T05:55:23,942 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T05:55:23,942 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true Metrics about Tables on a single HBase RegionServer 2024-11-13T05:55:23,944 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:55:23,944 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T05:55:23,944 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T05:55:23,944 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T05:55:25,827 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:55:25,828 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:55:25,829 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:55:25,829 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38823,1731477314332 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4171 ms java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T05:55:25,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:55:25,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:55:25,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:55:25,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:55:29,867 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:55:29,867 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:55:29,868 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:55:29,869 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38823,1731477314332 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8210 ms java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T05:55:29,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:55:29,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:55:29,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:55:29,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:55:39,927 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:55:39,928 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:55:39,928 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:55:39,929 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38823,1731477314332 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18270 ms java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T05:55:39,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:55:39,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:55:39,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:55:39,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:55:42,691 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:55:49,593 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=bc386b7bf2fe:37807 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-13T05:55:49,977 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:55:49,978 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:55:49,979 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:55:49,980 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38823,1731477314332 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28322 ms java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T05:55:49,981 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:55:49,981 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:55:49,981 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:55:49,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:55:57,236 INFO [master/bc386b7bf2fe:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-13T05:55:57,236 INFO [master/bc386b7bf2fe:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-13T05:56:00,047 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:56:00,047 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:56:00,049 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:56:00,049 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38823,1731477314332 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38391 ms java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T05:56:00,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:56:00,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:56:00,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:56:00,051 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:56:08,228 WARN [RS:2;bc386b7bf2fe:40327 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of bc386b7bf2fe,38823,1731477314332: status=status unset, state=RUNNING, startTime=1731477321632, completionTime=-1 appears to have been leaked 2024-11-13T05:56:10,108 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:56:10,108 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:56:10,111 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:56:10,112 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38823,1731477314332 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48453 ms java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T05:56:10,113 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:56:10,113 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:56:10,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:56:10,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:56:12,691 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:56:18,615 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a840e8f158b2eaf5c08595bd38db426e changed from -1.0 to 0.0, refreshing cache 2024-11-13T05:56:18,616 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1a492bcf61cc6ef0604137033fc7c90b changed from -1.0 to 0.0, refreshing cache 2024-11-13T05:56:18,616 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 89e8c9d6d5af67c75803894f4aea8ecc changed from -1.0 to 0.0, refreshing cache 2024-11-13T05:56:18,617 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 10c268d6da33b0a4590a478c0f32fd70 changed from -1.0 to 0.0, refreshing cache 2024-11-13T05:56:18,617 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 43fd53cccd0bd76519c89a78d6bf6131 changed from -1.0 to 0.0, refreshing cache 2024-11-13T05:56:18,617 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9bd21adcd80ce6ee8c5ed1a581c04749 changed from -1.0 to 0.0, refreshing cache 2024-11-13T05:56:18,618 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e86ecaa5f83edede0f153db264078a94 changed from -1.0 to 0.0, refreshing cache 2024-11-13T05:56:18,618 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 80710108d60e6f4a1b8975ccfe51e50e changed from -1.0 to 0.0, refreshing cache 2024-11-13T05:56:18,618 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 200f36be3ca8364314512c33e4ae22d6 changed from -1.0 to 0.0, refreshing cache 2024-11-13T05:56:18,619 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 19321613b69704e42207868d450f785b changed from -1.0 to 0.0, refreshing cache 2024-11-13T05:56:18,619 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 408b59e776ffab17a3ee0e8fb465c67c changed from -1.0 to 0.0, refreshing cache 2024-11-13T05:56:18,619 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b540ac2e8c4575b424d481848c4b4e52 changed from -1.0 to 0.0, refreshing cache 2024-11-13T05:56:18,620 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f7c7452ce5410ec8dcf456adb9b7db6b changed from -1.0 to 0.0, refreshing cache 2024-11-13T05:56:18,620 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 408dfc042fd5f2d964a15b01fd44966c changed from -1.0 to 0.0, refreshing cache 2024-11-13T05:56:18,620 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2810327548325125e764254e87017046 changed from -1.0 to 0.0, refreshing cache 2024-11-13T05:56:18,620 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ff62c0b5be01546e2b1563e065750a3b changed from -1.0 to 0.0, refreshing cache 2024-11-13T05:56:18,621 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 90b03edd8043eabbad2ddbd32409c478 changed from -1.0 to 0.0, refreshing cache 2024-11-13T05:56:19,509 INFO [regionserver/bc386b7bf2fe:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-13T05:56:21,668 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-13T05:55:21.663Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:55:21.769Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:55:21.981Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:55:22.290Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:55:22.798Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:55:23.826Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:55:25.831Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:55:29.870Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:55:39.929Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:55:49.981Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:56:00.050Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:56:10.114Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-13T05:56:21,669 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 1 exec state=pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:56:21,669 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T05:56:21,669 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:56:21,669 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T05:56:21,669 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T05:56:21,670 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T05:56:21,670 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T05:56:21,670 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T05:56:21,670 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T05:56:21,671 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T05:56:21,673 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T05:56:21,673 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T05:56:21,673 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T05:56:21,674 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=false; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:56:21,674 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=false; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T05:56:21,674 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-13T05:56:21,674 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:56:21,675 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bc386b7bf2fe,38823,1731477314332, table=hbase:meta, region=1588230740 2024-11-13T05:56:21,675 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,38823,1731477314332, state=OPEN 2024-11-13T05:56:21,729 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:56:21,731 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:56:21,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:56:21,738 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:56:21,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:56:21,739 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:56:21,739 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:56:21,739 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:56:21,742 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731477411742 2024-11-13T05:56:21,743 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T05:56:21,743 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true, isMeta: true 2024-11-13T05:56:21,743 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:56:21,743 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:56:21,744 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:56:21,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:56:21,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:56:21,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:56:21,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:56:21,747 INFO [PEWorker-1 {}] master.SplitWALManager(105): bc386b7bf2fe,38823,1731477314332 WAL count=1, meta=true 2024-11-13T05:56:21,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta}] 2024-11-13T05:56:21,748 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T05:56:21,756 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:56:21,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448}] 2024-11-13T05:56:21,847 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:56:21,847 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:56:21,848 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:38823 this server is in the failed servers list 2024-11-13T05:56:21,848 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:56:21,848 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:56:21,848 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:56:21,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45383 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-13T05:56:21,947 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, size=52.9 K (54214bytes) 2024-11-13T05:56:21,948 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-13T05:56:21,948 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-13T05:56:21,948 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta 2024-11-13T05:56:21,952 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta after 3ms 2024-11-13T05:56:21,960 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta took 14ms 2024-11-13T05:56:21,976 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 2024-11-13T05:56:21,983 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta.temp 2024-11-13T05:56:21,985 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta so closing down 2024-11-13T05:56:21,986 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T05:56:21,986 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta.temp 2024-11-13T05:56:21,991 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T05:56:21,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741864_1040 (size=53973) 2024-11-13T05:56:21,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741864_1040 (size=53973) 2024-11-13T05:56:21,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741864_1040 (size=53973) 2024-11-13T05:56:22,001 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) 2024-11-13T05:56:22,003 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta.temp to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-13T05:56:22,006 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 34 ms; skipped=1; WAL=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, size=52.9 K, length=54214, corrupted=false, cancelled=false 2024-11-13T05:56:22,007 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, journal: Splitting hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, size=52.9 K (54214bytes) at 1731477381947Finishing writing output for hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta so closing down at 1731477381985 (+38 ms)Creating recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta.temp at 1731477381986 (+1 ms)3 split writer threads finished at 1731477381991 (+5 ms)Closed recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) at 1731477382001 (+10 ms)Rename recovered edits hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta.temp to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1731477382003 (+2 ms)Processed 58 edits across 1 Regions in 34 ms; skipped=1; WAL=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, size=52.9 K, length=54214, corrupted=false, cancelled=false at 1731477382007 (+4 ms) 2024-11-13T05:56:22,007 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta 2024-11-13T05:56:22,008 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-13T05:56:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37807 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-13T05:56:22,017 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/oldWALs 2024-11-13T05:56:22,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-13T05:56:22,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 in 258 msec 2024-11-13T05:56:22,021 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:56:22,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-13T05:56:22,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 in 274 msec 2024-11-13T05:56:22,027 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting dir is empty, no logs to split. 2024-11-13T05:56:22,027 INFO [PEWorker-1 {}] master.SplitWALManager(105): bc386b7bf2fe,38823,1731477314332 WAL count=0, meta=true 2024-11-13T05:56:22,027 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if bc386b7bf2fe,38823,1731477314332 WAL splitting is done? wals=0, meta=true 2024-11-13T05:56:22,027 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T05:56:22,027 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:56:22,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:56:22,027 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:56:22,057 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:56:22,057 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:56:22,058 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:38823 this server is in the failed servers list 2024-11-13T05:56:22,058 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:56:22,058 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:56:22,058 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:56:22,367 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:56:22,367 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:56:22,368 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:38823 this server is in the failed servers list 2024-11-13T05:56:22,368 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:56:22,368 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:56:22,369 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:56:22,595 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=bc386b7bf2fe:37807 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-13T05:56:22,878 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:56:22,878 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:56:22,878 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:38823 this server is in the failed servers list 2024-11-13T05:56:22,879 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:56:22,879 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:56:22,879 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:56:23,887 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:56:23,887 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:56:23,889 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:56:23,890 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38823,1731477314332 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2146 ms java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T05:56:23,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:56:23,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:56:23,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:56:23,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:56:25,907 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:56:25,908 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:56:25,910 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:56:25,911 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38823,1731477314332 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4167 ms java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T05:56:25,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:56:25,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:56:25,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:56:25,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:56:29,928 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:56:29,928 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:56:29,929 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:56:29,930 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38823,1731477314332 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8186 ms java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T05:56:29,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:56:29,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:56:29,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:56:29,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:56:39,938 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:56:39,938 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:56:39,940 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:56:39,942 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38823,1731477314332 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18198 ms java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T05:56:39,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:56:39,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:56:39,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:56:39,944 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:56:42,692 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:56:50,017 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:56:50,017 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:56:50,019 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:56:50,021 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38823,1731477314332 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28277 ms java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T05:56:50,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:56:50,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:56:50,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:56:50,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:00,117 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:57:00,117 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:57:00,119 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:57:00,120 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38823,1731477314332 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38376 ms java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T05:57:00,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:00,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:57:00,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:57:00,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:10,167 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:57:10,167 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:57:10,170 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:57:10,171 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38823,1731477314332 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48427 ms java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T05:57:10,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:10,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:57:10,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:57:10,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:12,692 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:57:15,854 INFO [regionserver/bc386b7bf2fe:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactedHFilesCleaner was stopped 2024-11-13T05:57:21,748 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-13T05:56:21.745Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:56:21.848Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:56:22.058Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:56:22.369Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:56:22.879Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:56:23.891Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:56:25.913Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:56:29.931Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:56:39.943Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:56:50.022Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:00.122Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:10.172Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-13T05:57:21,751 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 2 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:57:21,751 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T05:57:21,751 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:57:21,751 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T05:57:21,751 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T05:57:21,752 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T05:57:21,752 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T05:57:21,752 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T05:57:21,753 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T05:57:21,755 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T05:57:21,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T05:57:21,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T05:57:21,759 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T05:57:21,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T05:57:21,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T05:57:21,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=false; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:57:21,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:57:21,761 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:57:21,761 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=false; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T05:57:21,761 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:57:21,761 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:57:21,761 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-13T05:57:21,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:57:21,762 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bc386b7bf2fe,38823,1731477314332, table=hbase:meta, region=1588230740 2024-11-13T05:57:21,762 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,38823,1731477314332, state=OPEN 2024-11-13T05:57:21,770 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:57:21,770 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:57:21,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:57:21,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:57:21,823 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:57:21,823 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:57:21,823 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:57:21,823 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:57:21,825 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731477471825 2024-11-13T05:57:21,825 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T05:57:21,826 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:57:21,826 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:57:21,828 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:57:21,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:21,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:57:21,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:57:21,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:21,830 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting dir is empty, no logs to split. 2024-11-13T05:57:21,830 INFO [PEWorker-1 {}] master.SplitWALManager(105): bc386b7bf2fe,38823,1731477314332 WAL count=0, meta=true 2024-11-13T05:57:21,830 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if bc386b7bf2fe,38823,1731477314332 WAL splitting is done? wals=0, meta=true 2024-11-13T05:57:21,830 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T05:57:21,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T05:57:21,833 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T05:57:21,833 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:57:21,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:57:21,833 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:57:21,937 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:57:21,938 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:57:21,938 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:38823 this server is in the failed servers list 2024-11-13T05:57:21,939 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:57:21,939 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:57:21,939 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:57:22,147 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:57:22,147 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:57:22,148 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:38823 this server is in the failed servers list 2024-11-13T05:57:22,148 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:57:22,148 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:57:22,149 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:57:22,457 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:57:22,458 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:57:22,458 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:38823 this server is in the failed servers list 2024-11-13T05:57:22,459 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:57:22,459 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:57:22,459 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:57:22,967 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:57:22,967 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:57:22,968 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:38823 this server is in the failed servers list 2024-11-13T05:57:22,968 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:57:22,968 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:57:22,968 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:57:23,978 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:57:23,978 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:57:23,979 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:57:23,981 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38823,1731477314332 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2154 ms java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T05:57:23,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:23,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:57:23,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:57:23,982 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:26,008 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:57:26,008 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:57:26,010 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:57:26,011 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38823,1731477314332 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4185 ms java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T05:57:26,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:26,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:57:26,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:57:26,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:30,037 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:57:30,037 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:57:30,039 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:57:30,040 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38823,1731477314332 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8214 ms java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T05:57:30,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:30,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:57:30,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:57:30,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:30,596 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=bc386b7bf2fe:37807 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-13T05:57:40,138 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:57:40,138 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:57:40,141 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:57:40,142 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38823,1731477314332 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18316 ms java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T05:57:40,143 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:40,143 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:57:40,144 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:57:40,144 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:42,693 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:57:50,197 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:57:50,197 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:57:50,200 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:57:50,201 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38823,1731477314332 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28375 ms java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T05:57:50,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:50,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:57:50,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:57:50,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:58:00,278 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:58:00,278 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:58:00,279 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:58:00,279 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38823,1731477314332 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38453 ms java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T05:58:00,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:58:00,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:58:00,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:58:00,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:58:10,347 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:58:10,347 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:58:10,348 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:58:10,348 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to bc386b7bf2fe,38823,1731477314332 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48522 ms java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-13T05:58:10,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:58:10,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:58:10,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:58:10,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:58:12,693 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T05:58:15,162 WARN [RS:2;bc386b7bf2fe:40327 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of bc386b7bf2fe,38823,1731477314332: status=Processing ServerCrashProcedure of bc386b7bf2fe,38823,1731477314332 current State SERVER_CRASH_SPLIT_META_LOGS, state=RUNNING, startTime=1731477321648, completionTime=-1 appears to have been leaked 2024-11-13T05:58:15,162 WARN [RS:2;bc386b7bf2fe:40327 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of bc386b7bf2fe,38823,1731477314332: status=Processing ServerCrashProcedure of bc386b7bf2fe,38823,1731477314332 current State SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, state=RUNNING, startTime=1731477381673, completionTime=-1 appears to have been leaked 2024-11-13T05:58:21,829 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-13T05:57:21.829Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:21.939Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:57:22.149Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:57:22.459Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:57:22.968Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:57:23.982Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:26.012Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:30.041Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:40.144Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:57:50.202Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:58:00.280Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:58:10.349Z, java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-13T05:58:21,833 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 3 exec state=pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:58:21,833 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T05:58:21,833 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:58:21,834 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T05:58:21,834 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T05:58:21,834 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T05:58:21,834 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T05:58:21,834 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T05:58:21,835 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T05:58:21,835 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T05:58:21,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T05:58:21,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T05:58:21,837 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T05:58:21,837 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T05:58:21,838 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T05:58:21,838 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=false; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:58:21,838 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:21,839 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:21,839 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=false; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T05:58:21,839 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:21,839 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:21,839 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-13T05:58:21,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:21,840 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bc386b7bf2fe,38823,1731477314332, table=hbase:meta, region=1588230740 2024-11-13T05:58:21,840 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,38823,1731477314332, state=OPEN 2024-11-13T05:58:21,842 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:58:21,842 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:58:21,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:21,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:21,907 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:21,907 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:21,907 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:21,907 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:21,909 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731477531909 2024-11-13T05:58:21,909 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T05:58:21,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T05:58:21,910 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T05:58:21,910 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:58:21,910 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:58:21,912 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:58:21,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:58:21,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:58:21,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:58:21,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:58:21,914 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:58:21,916 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T05:58:22,017 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:58:22,018 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1] 2024-11-13T05:58:22,018 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:38823 this server is in the failed servers list 2024-11-13T05:58:22,019 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:58:22,019 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:58:22,019 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:58:22,066 DEBUG [bc386b7bf2fe:37807 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-13T05:58:22,068 DEBUG [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(204): Hosts are {bc386b7bf2fe=0} racks are {/default-rack=0} 2024-11-13T05:58:22,068 DEBUG [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T05:58:22,068 DEBUG [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T05:58:22,069 DEBUG [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T05:58:22,069 DEBUG [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T05:58:22,069 INFO [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T05:58:22,069 INFO [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T05:58:22,069 DEBUG [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T05:58:22,070 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:22,073 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,45383,1731477314448, state=OPENING 2024-11-13T05:58:22,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:22,098 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:22,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:22,098 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:58:22,098 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:22,098 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:22,098 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:22,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,45383,1731477314448}] 2024-11-13T05:58:22,227 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:58:22,227 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45383,1731477314448, seqNum=-1] 2024-11-13T05:58:22,228 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:58:22,233 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58601, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:58:22,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45383 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Scan size: 121 connection: 172.17.0.2:58601 deadline: 1731477561915, exception=org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:22,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45383,1731477314448, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45383,1731477314448, seqNum=-1, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on bc386b7bf2fe,45383,1731477314448 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:58:22,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45383,1731477314448, seqNum=-1 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on bc386b7bf2fe,45383,1731477314448 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:58:22,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45383,1731477314448, seqNum=-1 from cache 2024-11-13T05:58:22,253 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50805, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:58:22,260 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T05:58:22,260 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:58:22,260 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T05:58:22,263 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bc386b7bf2fe%2C45383%2C1731477314448.meta, suffix=.meta, logDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,45383,1731477314448, archiveDir=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/oldWALs, maxLogs=32 2024-11-13T05:58:22,277 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,45383,1731477314448/bc386b7bf2fe%2C45383%2C1731477314448.meta.1731477502265.meta, exclude list is [], retry=0 2024-11-13T05:58:22,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41907,DS-b1ba899f-55cb-4657-847b-885d11a94025,DISK] 2024-11-13T05:58:22,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35961,DS-ffc916fc-f85e-4220-8961-748ff0692398,DISK] 2024-11-13T05:58:22,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37319,DS-f31c5c41-3c6c-4292-9e70-bf8ba9f019f7,DISK] 2024-11-13T05:58:22,282 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,45383,1731477314448/bc386b7bf2fe%2C45383%2C1731477314448.meta.1731477502265.meta 2024-11-13T05:58:22,282 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37481:37481),(127.0.0.1/127.0.0.1:46875:46875),(127.0.0.1/127.0.0.1:39447:39447)] 2024-11-13T05:58:22,282 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:58:22,283 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T05:58:22,283 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T05:58:22,283 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T05:58:22,283 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T05:58:22,283 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:58:22,283 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T05:58:22,283 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T05:58:22,285 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T05:58:22,286 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:58:22,286 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:22,287 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:58:22,287 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T05:58:22,288 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:58:22,288 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:22,288 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:58:22,288 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T05:58:22,289 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:58:22,289 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:22,289 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:58:22,289 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T05:58:22,290 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:58:22,290 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:22,291 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:58:22,291 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:58:22,292 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740 2024-11-13T05:58:22,294 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740 2024-11-13T05:58:22,294 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-13T05:58:22,307 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-13T05:58:22,309 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=49.58 KB heapSize=69.98 KB 2024-11-13T05:58:22,384 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/.tmp/info/35aa5624104944f19df45bbed1beedb8 is 268, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6./info:regioninfo/1731477317609/Put/seqid=0 2024-11-13T05:58:22,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741866_1042 (size=58053) 2024-11-13T05:58:22,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741866_1042 (size=58053) 2024-11-13T05:58:22,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741866_1042 (size=58053) 2024-11-13T05:58:22,395 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.31 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/.tmp/info/35aa5624104944f19df45bbed1beedb8 2024-11-13T05:58:22,462 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/.tmp/ns/9f9a4e4d7c5343f0b037c71c6f8187fe is 43, key is default/ns:d/1731477316624/Put/seqid=0 2024-11-13T05:58:22,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741867_1043 (size=5153) 2024-11-13T05:58:22,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741867_1043 (size=5153) 2024-11-13T05:58:22,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741867_1043 (size=5153) 2024-11-13T05:58:22,471 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/.tmp/ns/9f9a4e4d7c5343f0b037c71c6f8187fe 2024-11-13T05:58:22,494 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/.tmp/table/6803b5b5348546989da821bcb9230596 is 105, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/table:state/1731477318066/Put/seqid=0 2024-11-13T05:58:22,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741868_1044 (size=5620) 2024-11-13T05:58:22,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741868_1044 (size=5620) 2024-11-13T05:58:22,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741868_1044 (size=5620) 2024-11-13T05:58:22,521 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=202 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/.tmp/table/6803b5b5348546989da821bcb9230596 2024-11-13T05:58:22,533 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/.tmp/info/35aa5624104944f19df45bbed1beedb8 as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/info/35aa5624104944f19df45bbed1beedb8 2024-11-13T05:58:22,543 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/info/35aa5624104944f19df45bbed1beedb8, entries=260, sequenceid=60, filesize=56.7 K 2024-11-13T05:58:22,547 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/.tmp/ns/9f9a4e4d7c5343f0b037c71c6f8187fe as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/ns/9f9a4e4d7c5343f0b037c71c6f8187fe 2024-11-13T05:58:22,547 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:58:22,547 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45383,1731477314448, seqNum=-1] 2024-11-13T05:58:22,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45383 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 121 connection: 172.17.0.2:58601 deadline: 1731477561910, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:22,549 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45383,1731477314448, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45383,1731477314448, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on bc386b7bf2fe,45383,1731477314448 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:58:22,549 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45383,1731477314448, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on bc386b7bf2fe,45383,1731477314448 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T05:58:22,549 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45383,1731477314448, seqNum=-1 from cache 2024-11-13T05:58:22,556 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/ns/9f9a4e4d7c5343f0b037c71c6f8187fe, entries=2, sequenceid=60, filesize=5.0 K 2024-11-13T05:58:22,557 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/.tmp/table/6803b5b5348546989da821bcb9230596 as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/table/6803b5b5348546989da821bcb9230596 2024-11-13T05:58:22,565 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/table/6803b5b5348546989da821bcb9230596, entries=2, sequenceid=60, filesize=5.5 K 2024-11-13T05:58:22,569 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~49.58 KB/50772, heapSize ~69.69 KB/71360, currentSize=0 B/0 for 1588230740 in 259ms, sequenceid=60, compaction requested=false; wal=null 2024-11-13T05:58:22,571 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-13T05:58:22,572 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:58:22,572 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:58:22,573 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:58:22,574 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:58:22,577 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-13T05:58:22,578 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61184261, jitterRate=-0.0882834643125534}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:58:22,578 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T05:58:22,578 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731477502283Writing region info on filesystem at 1731477502283Initializing all the Stores at 1731477502284 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477502285 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477502285Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477502285Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477502285Obtaining lock to block concurrent updates at 1731477502310 (+25 ms)Preparing flush snapshotting stores in 1588230740 at 1731477502310Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=50772, getHeapSize=71600, getOffHeapSize=0, getCellsCount=264 at 1731477502313 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731477502313Flushing 1588230740/info: creating writer at 1731477502317 (+4 ms)Flushing 1588230740/info: appending metadata at 1731477502375 (+58 ms)Flushing 1588230740/info: closing flushed file at 1731477502378 (+3 ms)Flushing 1588230740/ns: creating writer at 1731477502440 (+62 ms)Flushing 1588230740/ns: appending metadata at 1731477502460 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1731477502461 (+1 ms)Flushing 1588230740/table: creating writer at 1731477502478 (+17 ms)Flushing 1588230740/table: appending metadata at 1731477502493 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731477502493Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@537f3db0: reopening flushed file at 1731477502531 (+38 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1515c113: reopening flushed file at 1731477502545 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75bf79da: reopening flushed file at 1731477502556 (+11 ms)Finished flush of dataSize ~49.58 KB/50772, heapSize ~69.69 KB/71360, currentSize=0 B/0 for 1588230740 in 259ms, sequenceid=60, compaction requested=false; wal=null at 1731477502569 (+13 ms)Cleaning up temporary data from old regions at 1731477502572 (+3 ms)Running coprocessor post-open hooks at 1731477502578 (+6 ms)Region opened successfully at 1731477502578 2024-11-13T05:58:22,580 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1731477502252 2024-11-13T05:58:22,582 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T05:58:22,582 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T05:58:22,583 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:22,584 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,45383,1731477314448, state=OPEN 2024-11-13T05:58:22,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:22,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:22,598 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:22,598 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:22,598 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:22,598 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:22,598 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:22,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-13T05:58:22,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,45383,1731477314448 in 500 msec 2024-11-13T05:58:22,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-13T05:58:22,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 691 msec 2024-11-13T05:58:23,058 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:58:23,058 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45383,1731477314448, seqNum=-1] 2024-11-13T05:58:23,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84316f2c5b758522338a907d7e8f386d, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=200f36be3ca8364314512c33e4ae22d6, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a840e8f158b2eaf5c08595bd38db426e, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e903b94793c361b461207b7aa3c133ac, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19321613b69704e42207868d450f785b, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a492bcf61cc6ef0604137033fc7c90b, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11abc6d6caf592b351052ae8adf50fd1, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=408b59e776ffab17a3ee0e8fb465c67c, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89e8c9d6d5af67c75803894f4aea8ecc, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b613e7fdd93a34284c913bfa5a7b222, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b540ac2e8c4575b424d481848c4b4e52, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10c268d6da33b0a4590a478c0f32fd70, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ff24ce91fac66115879ac4ec57a346a, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c7452ce5410ec8dcf456adb9b7db6b, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43fd53cccd0bd76519c89a78d6bf6131, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9afb90ba9af49bb3547d4438986e1fca, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=408dfc042fd5f2d964a15b01fd44966c, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bd21adcd80ce6ee8c5ed1a581c04749, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82c0a84200b3abe2cb346258143147c7, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2810327548325125e764254e87017046, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e86ecaa5f83edede0f153db264078a94, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a65241dbe6c2863f31a0138a73130225, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff62c0b5be01546e2b1563e065750a3b, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80710108d60e6f4a1b8975ccfe51e50e, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98cb12bd0a37bbbfee08e3459a85f735, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90b03edd8043eabbad2ddbd32409c478, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,093 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T05:58:23,093 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731477563093 2024-11-13T05:58:23,093 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731477623093 2024-11-13T05:58:23,093 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 1.1830 sec 2024-11-13T05:58:23,094 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): bc386b7bf2fe,38823,1731477314332 had 10 regions 2024-11-13T05:58:23,094 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T05:58:23,094 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:58:23,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:58:23,094 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:58:23,343 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 4 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:58:23,344 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T05:58:23,344 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:58:23,344 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T05:58:23,344 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T05:58:23,344 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T05:58:23,344 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T05:58:23,345 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T05:58:23,345 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T05:58:23,346 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T05:58:23,346 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:58:23,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T05:58:23,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T05:58:23,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T05:58:23,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T05:58:23,349 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T05:58:23,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T05:58:23,350 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T05:58:23,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, hasLock=false; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:58:23,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:23,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:23,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:58:23,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:23,351 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, hasLock=false; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T05:58:23,351 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:23,351 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:23,351 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:23,351 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:23,351 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-13T05:58:23,352 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=61 2024-11-13T05:58:23,352 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bc386b7bf2fe,45383,1731477314448, table=hbase:meta, region=1588230740 2024-11-13T05:58:23,352 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,45383,1731477314448, state=OPEN 2024-11-13T05:58:23,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:23,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:23,406 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:23,406 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:23,406 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:23,406 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:23,407 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731477533407 2024-11-13T05:58:23,407 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T05:58:23,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84316f2c5b758522338a907d7e8f386d, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=200f36be3ca8364314512c33e4ae22d6, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a840e8f158b2eaf5c08595bd38db426e, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e903b94793c361b461207b7aa3c133ac, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19321613b69704e42207868d450f785b, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,413 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a492bcf61cc6ef0604137033fc7c90b, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11abc6d6caf592b351052ae8adf50fd1, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=408b59e776ffab17a3ee0e8fb465c67c, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89e8c9d6d5af67c75803894f4aea8ecc, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b613e7fdd93a34284c913bfa5a7b222, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b540ac2e8c4575b424d481848c4b4e52, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10c268d6da33b0a4590a478c0f32fd70, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,414 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ff24ce91fac66115879ac4ec57a346a, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c7452ce5410ec8dcf456adb9b7db6b, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43fd53cccd0bd76519c89a78d6bf6131, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9afb90ba9af49bb3547d4438986e1fca, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=408dfc042fd5f2d964a15b01fd44966c, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bd21adcd80ce6ee8c5ed1a581c04749, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82c0a84200b3abe2cb346258143147c7, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,415 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2810327548325125e764254e87017046, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e86ecaa5f83edede0f153db264078a94, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a65241dbe6c2863f31a0138a73130225, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff62c0b5be01546e2b1563e065750a3b, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80710108d60e6f4a1b8975ccfe51e50e, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98cb12bd0a37bbbfee08e3459a85f735, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,416 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90b03edd8043eabbad2ddbd32409c478, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,416 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T05:58:23,416 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731477563416 2024-11-13T05:58:23,416 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731477623416 2024-11-13T05:58:23,416 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-13T05:58:23,417 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): bc386b7bf2fe,38823,1731477314332 had 9 regions 2024-11-13T05:58:23,417 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T05:58:23,418 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true, isMeta: false 2024-11-13T05:58:23,420 INFO [PEWorker-1 {}] master.SplitWALManager(105): bc386b7bf2fe,38823,1731477314332 WAL count=1, meta=false 2024-11-13T05:58:23,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.1731477316011}] 2024-11-13T05:58:23,420 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T05:58:23,420 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:58:23,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:58:23,421 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:58:23,667 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 5 exec state=pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:58:23,667 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T05:58:23,667 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:58:23,667 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T05:58:23,667 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T05:58:23,667 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T05:58:23,667 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T05:58:23,667 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T05:58:23,668 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T05:58:23,668 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T05:58:23,670 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T05:58:23,670 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T05:58:23,670 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T05:58:23,670 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T05:58:23,670 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T05:58:23,670 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T05:58:23,670 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T05:58:23,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=false; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:58:23,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:23,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:23,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:58:23,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:23,671 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=false; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T05:58:23,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:23,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:23,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:23,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:23,671 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-13T05:58:23,672 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=61 2024-11-13T05:58:23,672 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bc386b7bf2fe,45383,1731477314448, table=hbase:meta, region=1588230740 2024-11-13T05:58:23,672 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,45383,1731477314448, state=OPEN 2024-11-13T05:58:23,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:23,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:23,697 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:23,697 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:23,697 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:23,697 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:23,698 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731477533698 2024-11-13T05:58:23,698 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T05:58:23,704 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84316f2c5b758522338a907d7e8f386d, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=200f36be3ca8364314512c33e4ae22d6, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a840e8f158b2eaf5c08595bd38db426e, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e903b94793c361b461207b7aa3c133ac, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19321613b69704e42207868d450f785b, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a492bcf61cc6ef0604137033fc7c90b, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11abc6d6caf592b351052ae8adf50fd1, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=408b59e776ffab17a3ee0e8fb465c67c, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,705 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89e8c9d6d5af67c75803894f4aea8ecc, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b613e7fdd93a34284c913bfa5a7b222, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b540ac2e8c4575b424d481848c4b4e52, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10c268d6da33b0a4590a478c0f32fd70, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ff24ce91fac66115879ac4ec57a346a, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c7452ce5410ec8dcf456adb9b7db6b, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43fd53cccd0bd76519c89a78d6bf6131, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,706 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9afb90ba9af49bb3547d4438986e1fca, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,707 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=408dfc042fd5f2d964a15b01fd44966c, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,707 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bd21adcd80ce6ee8c5ed1a581c04749, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,707 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82c0a84200b3abe2cb346258143147c7, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,707 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2810327548325125e764254e87017046, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,707 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e86ecaa5f83edede0f153db264078a94, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,707 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a65241dbe6c2863f31a0138a73130225, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,707 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff62c0b5be01546e2b1563e065750a3b, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,707 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80710108d60e6f4a1b8975ccfe51e50e, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:23,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98cb12bd0a37bbbfee08e3459a85f735, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:23,708 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90b03edd8043eabbad2ddbd32409c478, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:23,708 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T05:58:23,708 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731477563708 2024-11-13T05:58:23,708 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731477623708 2024-11-13T05:58:23,708 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-13T05:58:23,708 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true, isMeta: false 2024-11-13T05:58:23,711 INFO [PEWorker-1 {}] master.SplitWALManager(105): bc386b7bf2fe,38823,1731477314332 WAL count=1, meta=false 2024-11-13T05:58:23,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.1731477316011}] 2024-11-13T05:58:23,711 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T05:58:23,713 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:23,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.1731477316011, worker=bc386b7bf2fe,40327,1731477314499}] 2024-11-13T05:58:23,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52829, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:58:23,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40327 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-13T05:58:23,892 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.1731477316011, size=327.1 K (334915bytes) 2024-11-13T05:58:23,892 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.1731477316011 2024-11-13T05:58:23,892 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.1731477316011 after 0ms 2024-11-13T05:58:23,896 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.1731477316011 took 5ms 2024-11-13T05:58:23,900 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 5ff24ce91fac66115879ac4ec57a346a: last_flushed_sequence_id: 18446744073709551615 2024-11-13T05:58:23,901 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 9afb90ba9af49bb3547d4438986e1fca: last_flushed_sequence_id: 18446744073709551615 2024-11-13T05:58:23,902 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for a65241dbe6c2863f31a0138a73130225: last_flushed_sequence_id: 18446744073709551615 2024-11-13T05:58:23,903 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 98cb12bd0a37bbbfee08e3459a85f735: last_flushed_sequence_id: 18446744073709551615 2024-11-13T05:58:23,904 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for e903b94793c361b461207b7aa3c133ac: last_flushed_sequence_id: 18446744073709551615 2024-11-13T05:58:23,906 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 11abc6d6caf592b351052ae8adf50fd1: last_flushed_sequence_id: 18446744073709551615 2024-11-13T05:58:23,907 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 82c0a84200b3abe2cb346258143147c7: last_flushed_sequence_id: 18446744073709551615 2024-11-13T05:58:23,908 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 84316f2c5b758522338a907d7e8f386d: last_flushed_sequence_id: 18446744073709551615 2024-11-13T05:58:23,909 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 4b613e7fdd93a34284c913bfa5a7b222: last_flushed_sequence_id: 18446744073709551615 2024-11-13T05:58:23,916 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp 2024-11-13T05:58:23,917 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp 2024-11-13T05:58:23,917 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp 2024-11-13T05:58:23,919 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp 2024-11-13T05:58:23,919 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp 2024-11-13T05:58:23,921 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp 2024-11-13T05:58:23,923 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp 2024-11-13T05:58:23,927 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4b613e7fdd93a34284c913bfa5a7b222/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp 2024-11-13T05:58:23,927 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp 2024-11-13T05:58:23,930 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.1731477316011 so closing down 2024-11-13T05:58:23,930 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T05:58:23,930 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp 2024-11-13T05:58:23,930 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp 2024-11-13T05:58:23,930 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp 2024-11-13T05:58:23,931 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp 2024-11-13T05:58:23,932 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp 2024-11-13T05:58:23,933 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp 2024-11-13T05:58:23,934 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp 2024-11-13T05:58:23,936 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp 2024-11-13T05:58:23,936 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp 2024-11-13T05:58:23,938 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T05:58:23,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741869_1045 (size=36767) 2024-11-13T05:58:23,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741869_1045 (size=36767) 2024-11-13T05:58:23,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741869_1045 (size=36767) 2024-11-13T05:58:23,951 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T05:58:23,953 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735/recovered.edits/0000000000000000004 2024-11-13T05:58:23,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741871_1047 (size=36767) 2024-11-13T05:58:23,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741871_1047 (size=36767) 2024-11-13T05:58:23,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741871_1047 (size=36767) 2024-11-13T05:58:23,966 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4b613e7fdd93a34284c913bfa5a7b222/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T05:58:23,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741870_1046 (size=36767) 2024-11-13T05:58:23,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741870_1046 (size=36767) 2024-11-13T05:58:23,969 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T05:58:23,969 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4b613e7fdd93a34284c913bfa5a7b222/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4b613e7fdd93a34284c913bfa5a7b222/recovered.edits/0000000000000000004 2024-11-13T05:58:23,971 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7/recovered.edits/0000000000000000004 2024-11-13T05:58:23,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741870_1046 (size=36767) 2024-11-13T05:58:23,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741872_1048 (size=36767) 2024-11-13T05:58:23,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741872_1048 (size=36767) 2024-11-13T05:58:23,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741872_1048 (size=36767) 2024-11-13T05:58:23,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741873_1049 (size=36767) 2024-11-13T05:58:23,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741874_1050 (size=36767) 2024-11-13T05:58:23,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741873_1049 (size=36767) 2024-11-13T05:58:23,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741874_1050 (size=36767) 2024-11-13T05:58:23,981 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T05:58:23,981 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T05:58:23,982 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac/recovered.edits/0000000000000000004 2024-11-13T05:58:23,983 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a/recovered.edits/0000000000000000004 2024-11-13T05:58:23,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741873_1049 (size=36767) 2024-11-13T05:58:23,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741874_1050 (size=36767) 2024-11-13T05:58:23,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741875_1051 (size=36767) 2024-11-13T05:58:23,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741876_1052 (size=36767) 2024-11-13T05:58:23,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741875_1051 (size=36767) 2024-11-13T05:58:23,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741876_1052 (size=36767) 2024-11-13T05:58:23,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741875_1051 (size=36767) 2024-11-13T05:58:23,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741876_1052 (size=36767) 2024-11-13T05:58:23,996 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T05:58:23,997 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-13T05:58:23,998 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1/recovered.edits/0000000000000000004 2024-11-13T05:58:23,998 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d/recovered.edits/0000000000000000004 2024-11-13T05:58:24,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741877_1053 (size=36767) 2024-11-13T05:58:24,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741877_1053 (size=36767) 2024-11-13T05:58:24,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741877_1053 (size=36767) 2024-11-13T05:58:24,005 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T05:58:24,007 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225/recovered.edits/0000000000000000004 2024-11-13T05:58:24,372 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-13T05:58:24,373 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca/recovered.edits/0000000000000000004 2024-11-13T05:58:24,373 INFO [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 18 edits across 9 Regions in 475 ms; skipped=9; WAL=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.1731477316011, size=327.1 K, length=334915, corrupted=false, cancelled=false 2024-11-13T05:58:24,374 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.1731477316011, journal: Splitting hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.1731477316011, size=327.1 K (334915bytes) at 1731477503892Creating recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp at 1731477503917 (+25 ms)Creating recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp at 1731477503919 (+2 ms)Creating recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp at 1731477503927 (+8 ms)Creating recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp at 1731477503927Split 9 edits, skipped 9 edits. at 1731477503929 (+2 ms)Finishing writing output for hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.1731477316011 so closing down at 1731477503930 (+1 ms)Creating recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp at 1731477503930Creating recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp at 1731477503932 (+2 ms)Creating recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp at 1731477503932Creating recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp at 1731477503936 (+4 ms)Creating recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp at 17314775039363 split writer threads finished at 1731477503938 (+2 ms)Closed recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731477503951 (+13 ms)Rename recovered edits hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735/recovered.edits/0000000000000000004 at 1731477503953 (+2 ms)Closed recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4b613e7fdd93a34284c913bfa5a7b222/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731477503966 (+13 ms)Closed recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731477503969 (+3 ms)Rename recovered edits hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4b613e7fdd93a34284c913bfa5a7b222/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4b613e7fdd93a34284c913bfa5a7b222/recovered.edits/0000000000000000004 at 1731477503969Rename recovered edits hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7/recovered.edits/0000000000000000004 at 1731477503971 (+2 ms)Closed recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731477503981 (+10 ms)Closed recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731477503981Rename recovered edits hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac/recovered.edits/0000000000000000004 at 1731477503982 (+1 ms)Rename recovered edits hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a/recovered.edits/0000000000000000004 at 1731477503983 (+1 ms)Closed recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731477503996 (+13 ms)Closed recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1731477503997 (+1 ms)Rename recovered edits hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1/recovered.edits/0000000000000000004 at 1731477503998 (+1 ms)Rename recovered edits hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d/recovered.edits/0000000000000000004 at 1731477503998Closed recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731477504005 (+7 ms)Rename recovered edits hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225/recovered.edits/0000000000000000004 at 1731477504007 (+2 ms)Closed recovered edits writer path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1731477504372 (+365 ms)Rename recovered edits hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca/recovered.edits/0000000000000000004-bc386b7bf2fe%2C38823%2C1731477314332.1731477316011.temp to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca/recovered.edits/0000000000000000004 at 1731477504373 (+1 ms)Processed 18 edits across 9 Regions in 475 ms; skipped=9; WAL=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.1731477316011, size=327.1 K, length=334915, corrupted=false, cancelled=false at 1731477504374 (+1 ms) 2024-11-13T05:58:24,374 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.1731477316011 2024-11-13T05:58:24,374 DEBUG [RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-13T05:58:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37807 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-13T05:58:24,378 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting/bc386b7bf2fe%2C38823%2C1731477314332.1731477316011 to hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/oldWALs 2024-11-13T05:58:24,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-13T05:58:24,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.1731477316011, worker=bc386b7bf2fe,40327,1731477314499 in 664 msec 2024-11-13T05:58:24,382 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:24,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-13T05:58:24,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.1731477316011, worker=bc386b7bf2fe,40327,1731477314499 in 671 msec 2024-11-13T05:58:24,385 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting dir is empty, no logs to split. 2024-11-13T05:58:24,385 INFO [PEWorker-1 {}] master.SplitWALManager(105): bc386b7bf2fe,38823,1731477314332 WAL count=0, meta=false 2024-11-13T05:58:24,385 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if bc386b7bf2fe,38823,1731477314332 WAL splitting is done? wals=0, meta=false 2024-11-13T05:58:24,386 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T05:58:24,386 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:58:24,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:58:24,386 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:58:24,458 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 6 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:58:24,459 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T05:58:24,459 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:58:24,459 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T05:58:24,459 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T05:58:24,460 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T05:58:24,460 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T05:58:24,460 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T05:58:24,461 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T05:58:24,462 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T05:58:24,463 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:58:24,466 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T05:58:24,466 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T05:58:24,466 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T05:58:24,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T05:58:24,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T05:58:24,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T05:58:24,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T05:58:24,467 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T05:58:24,467 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T05:58:24,468 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=false; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:58:24,468 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:24,468 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:24,468 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:58:24,468 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:24,468 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.1731477316011, worker=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:24,468 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.1731477316011, worker=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:24,468 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=false; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T05:58:24,468 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:24,468 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:24,469 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:24,469 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:24,469 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.1731477316011, worker=bc386b7bf2fe,40327,1731477314499 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:24,469 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.1731477316011, worker=bc386b7bf2fe,40327,1731477314499 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:24,469 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-13T05:58:24,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=61 2024-11-13T05:58:24,469 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bc386b7bf2fe,45383,1731477314448, table=hbase:meta, region=1588230740 2024-11-13T05:58:24,470 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,45383,1731477314448, state=OPEN 2024-11-13T05:58:24,472 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:58:24,523 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:24,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:24,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:24,523 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:24,523 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:24,523 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:24,524 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731477534524 2024-11-13T05:58:24,524 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T05:58:24,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84316f2c5b758522338a907d7e8f386d, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:24,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=200f36be3ca8364314512c33e4ae22d6, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:24,531 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a840e8f158b2eaf5c08595bd38db426e, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:24,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e903b94793c361b461207b7aa3c133ac, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:24,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19321613b69704e42207868d450f785b, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:24,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a492bcf61cc6ef0604137033fc7c90b, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:24,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11abc6d6caf592b351052ae8adf50fd1, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:24,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=408b59e776ffab17a3ee0e8fb465c67c, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:24,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89e8c9d6d5af67c75803894f4aea8ecc, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:24,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b613e7fdd93a34284c913bfa5a7b222, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:24,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b540ac2e8c4575b424d481848c4b4e52, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:24,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10c268d6da33b0a4590a478c0f32fd70, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:24,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ff24ce91fac66115879ac4ec57a346a, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:24,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c7452ce5410ec8dcf456adb9b7db6b, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:24,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43fd53cccd0bd76519c89a78d6bf6131, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:24,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9afb90ba9af49bb3547d4438986e1fca, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:24,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=408dfc042fd5f2d964a15b01fd44966c, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:24,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bd21adcd80ce6ee8c5ed1a581c04749, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:24,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82c0a84200b3abe2cb346258143147c7, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:24,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2810327548325125e764254e87017046, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:24,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e86ecaa5f83edede0f153db264078a94, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:24,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a65241dbe6c2863f31a0138a73130225, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:24,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff62c0b5be01546e2b1563e065750a3b, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:24,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80710108d60e6f4a1b8975ccfe51e50e, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:24,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98cb12bd0a37bbbfee08e3459a85f735, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:24,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90b03edd8043eabbad2ddbd32409c478, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:24,535 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T05:58:24,535 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731477564535 2024-11-13T05:58:24,535 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731477624535 2024-11-13T05:58:24,535 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-13T05:58:24,536 INFO [PEWorker-1 {}] master.MasterWalManager(331): Log dir for server bc386b7bf2fe,38823,1731477314332 does not exist 2024-11-13T05:58:24,536 INFO [PEWorker-1 {}] master.SplitWALManager(105): bc386b7bf2fe,38823,1731477314332 WAL count=0, meta=false 2024-11-13T05:58:24,536 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if bc386b7bf2fe,38823,1731477314332 WAL splitting is done? wals=0, meta=false 2024-11-13T05:58:24,537 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for bc386b7bf2fe,38823,1731477314332 failed, ignore...File hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/WALs/bc386b7bf2fe,38823,1731477314332-splitting does not exist. 2024-11-13T05:58:24,537 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T05:58:24,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=84316f2c5b758522338a907d7e8f386d, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4b613e7fdd93a34284c913bfa5a7b222, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=11abc6d6caf592b351052ae8adf50fd1, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=98cb12bd0a37bbbfee08e3459a85f735, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e903b94793c361b461207b7aa3c133ac, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ff24ce91fac66115879ac4ec57a346a, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a65241dbe6c2863f31a0138a73130225, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9afb90ba9af49bb3547d4438986e1fca, ASSIGN}, {pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=82c0a84200b3abe2cb346258143147c7, ASSIGN}] 2024-11-13T05:58:24,539 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T05:58:24,539 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:58:24,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:58:24,539 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:58:24,785 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 7 exec state=pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:58:24,785 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T05:58:24,785 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:58:24,785 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T05:58:24,785 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T05:58:24,786 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T05:58:24,786 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T05:58:24,786 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T05:58:24,786 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T05:58:24,788 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T05:58:24,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T05:58:24,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T05:58:24,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T05:58:24,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T05:58:24,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T05:58:24,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T05:58:24,791 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T05:58:24,791 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T05:58:24,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T05:58:24,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=false; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:58:24,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:24,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:24,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:58:24,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:24,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.1731477316011, worker=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:24,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.1731477316011, worker=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:24,794 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=false; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T05:58:24,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:24,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:24,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:24,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:24,795 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.1731477316011, worker=bc386b7bf2fe,40327,1731477314499 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:24,795 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.1731477316011, worker=bc386b7bf2fe,40327,1731477314499 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:24,795 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-13T05:58:24,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=61 2024-11-13T05:58:24,796 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bc386b7bf2fe,45383,1731477314448, table=hbase:meta, region=1588230740 2024-11-13T05:58:24,796 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,45383,1731477314448, state=OPEN 2024-11-13T05:58:24,830 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:58:24,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:24,830 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:24,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:24,830 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:24,830 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:24,830 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:24,831 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731477534831 2024-11-13T05:58:24,831 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T05:58:24,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84316f2c5b758522338a907d7e8f386d, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:24,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=200f36be3ca8364314512c33e4ae22d6, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:24,836 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a840e8f158b2eaf5c08595bd38db426e, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:24,837 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e903b94793c361b461207b7aa3c133ac, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:24,837 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19321613b69704e42207868d450f785b, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:24,837 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a492bcf61cc6ef0604137033fc7c90b, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:24,837 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11abc6d6caf592b351052ae8adf50fd1, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:24,837 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=408b59e776ffab17a3ee0e8fb465c67c, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:24,837 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89e8c9d6d5af67c75803894f4aea8ecc, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:24,837 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b613e7fdd93a34284c913bfa5a7b222, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:24,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b540ac2e8c4575b424d481848c4b4e52, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:24,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10c268d6da33b0a4590a478c0f32fd70, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:24,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ff24ce91fac66115879ac4ec57a346a, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:24,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c7452ce5410ec8dcf456adb9b7db6b, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:24,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43fd53cccd0bd76519c89a78d6bf6131, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:24,838 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9afb90ba9af49bb3547d4438986e1fca, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:24,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=408dfc042fd5f2d964a15b01fd44966c, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:24,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bd21adcd80ce6ee8c5ed1a581c04749, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:24,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82c0a84200b3abe2cb346258143147c7, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:24,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2810327548325125e764254e87017046, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:24,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e86ecaa5f83edede0f153db264078a94, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:24,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a65241dbe6c2863f31a0138a73130225, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:24,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff62c0b5be01546e2b1563e065750a3b, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:24,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80710108d60e6f4a1b8975ccfe51e50e, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:24,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98cb12bd0a37bbbfee08e3459a85f735, regionState=OPEN, lastHost=bc386b7bf2fe,38823,1731477314332, regionLocation=bc386b7bf2fe,38823,1731477314332, openSeqNum=2 2024-11-13T05:58:24,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90b03edd8043eabbad2ddbd32409c478, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:24,840 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T05:58:24,840 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731477564840 2024-11-13T05:58:24,840 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731477624840 2024-11-13T05:58:24,840 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-13T05:58:24,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=84316f2c5b758522338a907d7e8f386d, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4b613e7fdd93a34284c913bfa5a7b222, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=11abc6d6caf592b351052ae8adf50fd1, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=98cb12bd0a37bbbfee08e3459a85f735, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e903b94793c361b461207b7aa3c133ac, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ff24ce91fac66115879ac4ec57a346a, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a65241dbe6c2863f31a0138a73130225, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9afb90ba9af49bb3547d4438986e1fca, ASSIGN}, {pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=82c0a84200b3abe2cb346258143147c7, ASSIGN}] 2024-11-13T05:58:24,841 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T05:58:24,844 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=82c0a84200b3abe2cb346258143147c7, ASSIGN 2024-11-13T05:58:24,846 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=82c0a84200b3abe2cb346258143147c7, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T05:58:24,847 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9afb90ba9af49bb3547d4438986e1fca, ASSIGN 2024-11-13T05:58:24,847 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9afb90ba9af49bb3547d4438986e1fca, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T05:58:24,848 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a65241dbe6c2863f31a0138a73130225, ASSIGN 2024-11-13T05:58:24,849 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a65241dbe6c2863f31a0138a73130225, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T05:58:24,850 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ff24ce91fac66115879ac4ec57a346a, ASSIGN 2024-11-13T05:58:24,850 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ff24ce91fac66115879ac4ec57a346a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T05:58:24,851 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e903b94793c361b461207b7aa3c133ac, ASSIGN 2024-11-13T05:58:24,852 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e903b94793c361b461207b7aa3c133ac, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T05:58:24,853 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=98cb12bd0a37bbbfee08e3459a85f735, ASSIGN 2024-11-13T05:58:24,853 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=98cb12bd0a37bbbfee08e3459a85f735, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T05:58:24,854 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=11abc6d6caf592b351052ae8adf50fd1, ASSIGN 2024-11-13T05:58:24,855 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=11abc6d6caf592b351052ae8adf50fd1, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T05:58:24,856 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4b613e7fdd93a34284c913bfa5a7b222, ASSIGN 2024-11-13T05:58:24,856 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4b613e7fdd93a34284c913bfa5a7b222, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T05:58:24,857 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=84316f2c5b758522338a907d7e8f386d, ASSIGN 2024-11-13T05:58:24,857 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=84316f2c5b758522338a907d7e8f386d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T05:58:24,996 DEBUG [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(204): Hosts are {bc386b7bf2fe=0} racks are {/default-rack=0} 2024-11-13T05:58:24,997 DEBUG [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T05:58:24,997 DEBUG [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T05:58:24,997 DEBUG [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T05:58:24,997 DEBUG [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T05:58:24,997 INFO [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T05:58:24,997 INFO [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T05:58:24,997 DEBUG [bc386b7bf2fe:37807 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T05:58:24,998 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=98cb12bd0a37bbbfee08e3459a85f735, regionState=OPENING, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:24,999 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=82c0a84200b3abe2cb346258143147c7, regionState=OPENING, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:24,999 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=5ff24ce91fac66115879ac4ec57a346a, regionState=OPENING, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:25,000 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=11abc6d6caf592b351052ae8adf50fd1, regionState=OPENING, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:25,001 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=84316f2c5b758522338a907d7e8f386d, regionState=OPENING, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:25,001 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=a65241dbe6c2863f31a0138a73130225, regionState=OPENING, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:25,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=98cb12bd0a37bbbfee08e3459a85f735, ASSIGN because future has completed 2024-11-13T05:58:25,002 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=9afb90ba9af49bb3547d4438986e1fca, regionState=OPENING, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:25,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 98cb12bd0a37bbbfee08e3459a85f735, server=bc386b7bf2fe,40327,1731477314499}] 2024-11-13T05:58:25,003 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=82c0a84200b3abe2cb346258143147c7, ASSIGN because future has completed 2024-11-13T05:58:25,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ff24ce91fac66115879ac4ec57a346a, ASSIGN because future has completed 2024-11-13T05:58:25,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=11abc6d6caf592b351052ae8adf50fd1, ASSIGN because future has completed 2024-11-13T05:58:25,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=84316f2c5b758522338a907d7e8f386d, ASSIGN because future has completed 2024-11-13T05:58:25,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 84316f2c5b758522338a907d7e8f386d, server=bc386b7bf2fe,40327,1731477314499}] 2024-11-13T05:58:25,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a65241dbe6c2863f31a0138a73130225, ASSIGN because future has completed 2024-11-13T05:58:25,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9afb90ba9af49bb3547d4438986e1fca, ASSIGN because future has completed 2024-11-13T05:58:25,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9afb90ba9af49bb3547d4438986e1fca, server=bc386b7bf2fe,45383,1731477314448}] 2024-11-13T05:58:25,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a65241dbe6c2863f31a0138a73130225, server=bc386b7bf2fe,45383,1731477314448}] 2024-11-13T05:58:25,015 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11abc6d6caf592b351052ae8adf50fd1, server=bc386b7bf2fe,40327,1731477314499}] 2024-11-13T05:58:25,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ff24ce91fac66115879ac4ec57a346a, server=bc386b7bf2fe,40327,1731477314499}] 2024-11-13T05:58:25,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82c0a84200b3abe2cb346258143147c7, server=bc386b7bf2fe,40327,1731477314499}] 2024-11-13T05:58:25,022 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=4b613e7fdd93a34284c913bfa5a7b222, regionState=OPENING, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:25,023 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=e903b94793c361b461207b7aa3c133ac, regionState=OPENING, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:25,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4b613e7fdd93a34284c913bfa5a7b222, ASSIGN because future has completed 2024-11-13T05:58:25,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b613e7fdd93a34284c913bfa5a7b222, server=bc386b7bf2fe,45383,1731477314448}] 2024-11-13T05:58:25,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e903b94793c361b461207b7aa3c133ac, ASSIGN because future has completed 2024-11-13T05:58:25,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e903b94793c361b461207b7aa3c133ac, server=bc386b7bf2fe,45383,1731477314448}] 2024-11-13T05:58:25,166 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735. 2024-11-13T05:58:25,166 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => 98cb12bd0a37bbbfee08e3459a85f735, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-13T05:58:25,167 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:58:25,167 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:58:25,167 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:58:25,167 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:58:25,168 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225. 2024-11-13T05:58:25,168 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => a65241dbe6c2863f31a0138a73130225, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-13T05:58:25,169 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true a65241dbe6c2863f31a0138a73130225 2024-11-13T05:58:25,169 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:58:25,169 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for a65241dbe6c2863f31a0138a73130225 2024-11-13T05:58:25,169 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for a65241dbe6c2863f31a0138a73130225 2024-11-13T05:58:25,169 INFO [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:58:25,170 INFO [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98cb12bd0a37bbbfee08e3459a85f735 columnFamilyName colfamily11 2024-11-13T05:58:25,170 DEBUG [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,171 INFO [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region a65241dbe6c2863f31a0138a73130225 2024-11-13T05:58:25,171 INFO [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] regionserver.HStore(327): Store=98cb12bd0a37bbbfee08e3459a85f735/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,171 INFO [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:58:25,172 INFO [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a65241dbe6c2863f31a0138a73130225 columnFamilyName colfamily11 2024-11-13T05:58:25,172 INFO [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98cb12bd0a37bbbfee08e3459a85f735 columnFamilyName colfamily21 2024-11-13T05:58:25,172 DEBUG [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,172 DEBUG [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,172 INFO [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] regionserver.HStore(327): Store=98cb12bd0a37bbbfee08e3459a85f735/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,172 INFO [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] regionserver.HStore(327): Store=a65241dbe6c2863f31a0138a73130225/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,172 INFO [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:58:25,172 INFO [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region a65241dbe6c2863f31a0138a73130225 2024-11-13T05:58:25,173 INFO [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98cb12bd0a37bbbfee08e3459a85f735 columnFamilyName colfamily31 2024-11-13T05:58:25,173 DEBUG [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,173 INFO [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a65241dbe6c2863f31a0138a73130225 columnFamilyName colfamily21 2024-11-13T05:58:25,174 DEBUG [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,174 INFO [StoreOpener-98cb12bd0a37bbbfee08e3459a85f735-1 {}] regionserver.HStore(327): Store=98cb12bd0a37bbbfee08e3459a85f735/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,174 INFO [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] regionserver.HStore(327): Store=a65241dbe6c2863f31a0138a73130225/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,174 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:58:25,174 INFO [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region a65241dbe6c2863f31a0138a73130225 2024-11-13T05:58:25,175 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:58:25,175 INFO [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a65241dbe6c2863f31a0138a73130225 columnFamilyName colfamily31 2024-11-13T05:58:25,175 DEBUG [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,176 INFO [StoreOpener-a65241dbe6c2863f31a0138a73130225-1 {}] regionserver.HStore(327): Store=a65241dbe6c2863f31a0138a73130225/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,176 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for a65241dbe6c2863f31a0138a73130225 2024-11-13T05:58:25,177 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225 2024-11-13T05:58:25,177 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:58:25,178 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735/recovered.edits/0000000000000000004 2024-11-13T05:58:25,179 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225 2024-11-13T05:58:25,180 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225/recovered.edits/0000000000000000004 2024-11-13T05:58:25,186 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735/recovered.edits/0000000000000000004 2024-11-13T05:58:25,186 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(2902): Flushing 98cb12bd0a37bbbfee08e3459a85f735 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:25,186 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225/recovered.edits/0000000000000000004 2024-11-13T05:58:25,186 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(2902): Flushing a65241dbe6c2863f31a0138a73130225 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:25,206 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225/.tmp/colfamily11/ff4006de5f744e25ae50b07a3fbd9308 is 52, key is vvv/colfamily11:colfamily11/1731477320796/Put/seqid=0 2024-11-13T05:58:25,210 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735/.tmp/colfamily11/03aa494c9c8b41388a2ac874b6c15f19 is 52, key is yyy/colfamily11:colfamily11/1731477320835/Put/seqid=0 2024-11-13T05:58:25,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741878_1054 (size=39263) 2024-11-13T05:58:25,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741878_1054 (size=39263) 2024-11-13T05:58:25,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741878_1054 (size=39263) 2024-11-13T05:58:25,215 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225/.tmp/colfamily11/ff4006de5f744e25ae50b07a3fbd9308 2024-11-13T05:58:25,222 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225/.tmp/colfamily11/ff4006de5f744e25ae50b07a3fbd9308 as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225/colfamily11/ff4006de5f744e25ae50b07a3fbd9308 2024-11-13T05:58:25,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741879_1055 (size=39263) 2024-11-13T05:58:25,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741879_1055 (size=39263) 2024-11-13T05:58:25,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741879_1055 (size=39263) 2024-11-13T05:58:25,226 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735/.tmp/colfamily11/03aa494c9c8b41388a2ac874b6c15f19 2024-11-13T05:58:25,231 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225/colfamily11/ff4006de5f744e25ae50b07a3fbd9308, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T05:58:25,231 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for a65241dbe6c2863f31a0138a73130225 in 45ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T05:58:25,232 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225/recovered.edits/0000000000000000004 2024-11-13T05:58:25,233 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for a65241dbe6c2863f31a0138a73130225 2024-11-13T05:58:25,233 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for a65241dbe6c2863f31a0138a73130225 2024-11-13T05:58:25,234 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:58:25,234 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735/.tmp/colfamily11/03aa494c9c8b41388a2ac874b6c15f19 as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735/colfamily11/03aa494c9c8b41388a2ac874b6c15f19 2024-11-13T05:58:25,235 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for a65241dbe6c2863f31a0138a73130225 2024-11-13T05:58:25,238 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T05:58:25,239 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened a65241dbe6c2863f31a0138a73130225; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72306640, jitterRate=0.0774528980255127}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:58:25,239 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a65241dbe6c2863f31a0138a73130225 2024-11-13T05:58:25,239 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for a65241dbe6c2863f31a0138a73130225: Running coprocessor pre-open hook at 1731477505169Writing region info on filesystem at 1731477505169Initializing all the Stores at 1731477505170 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505170Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505170Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505170Obtaining lock to block concurrent updates at 1731477505186 (+16 ms)Preparing flush snapshotting stores in a65241dbe6c2863f31a0138a73130225 at 1731477505186Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477505186Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225. at 1731477505186Flushing a65241dbe6c2863f31a0138a73130225/colfamily11: creating writer at 1731477505186Flushing a65241dbe6c2863f31a0138a73130225/colfamily11: appending metadata at 1731477505205 (+19 ms)Flushing a65241dbe6c2863f31a0138a73130225/colfamily11: closing flushed file at 1731477505205Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c1f2949: reopening flushed file at 1731477505221 (+16 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for a65241dbe6c2863f31a0138a73130225 in 45ms, sequenceid=4, compaction requested=false; wal=null at 1731477505231 (+10 ms)Cleaning up temporary data from old regions at 1731477505233 (+2 ms)Running coprocessor post-open hooks at 1731477505239 (+6 ms)Region opened successfully at 1731477505239 2024-11-13T05:58:25,241 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225., pid=76, masterSystemTime=1731477505161 2024-11-13T05:58:25,243 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735/colfamily11/03aa494c9c8b41388a2ac874b6c15f19, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T05:58:25,243 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 98cb12bd0a37bbbfee08e3459a85f735 in 57ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T05:58:25,243 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225. 2024-11-13T05:58:25,243 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225. 2024-11-13T05:58:25,243 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac. 2024-11-13T05:58:25,244 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => e903b94793c361b461207b7aa3c133ac, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-13T05:58:25,244 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true e903b94793c361b461207b7aa3c133ac 2024-11-13T05:58:25,244 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:58:25,244 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735/recovered.edits/0000000000000000004 2024-11-13T05:58:25,244 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for e903b94793c361b461207b7aa3c133ac 2024-11-13T05:58:25,244 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for e903b94793c361b461207b7aa3c133ac 2024-11-13T05:58:25,244 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=a65241dbe6c2863f31a0138a73130225, regionState=OPEN, openSeqNum=5, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:25,245 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:58:25,245 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:58:25,246 INFO [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region e903b94793c361b461207b7aa3c133ac 2024-11-13T05:58:25,247 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:58:25,247 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a65241dbe6c2863f31a0138a73130225, server=bc386b7bf2fe,45383,1731477314448 because future has completed 2024-11-13T05:58:25,248 INFO [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e903b94793c361b461207b7aa3c133ac columnFamilyName colfamily11 2024-11-13T05:58:25,249 DEBUG [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,249 INFO [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] regionserver.HStore(327): Store=e903b94793c361b461207b7aa3c133ac/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,249 INFO [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region e903b94793c361b461207b7aa3c133ac 2024-11-13T05:58:25,249 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:58:25,250 INFO [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e903b94793c361b461207b7aa3c133ac columnFamilyName colfamily21 2024-11-13T05:58:25,251 DEBUG [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=70 2024-11-13T05:58:25,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure a65241dbe6c2863f31a0138a73130225, server=bc386b7bf2fe,45383,1731477314448 in 236 msec 2024-11-13T05:58:25,251 INFO [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] regionserver.HStore(327): Store=e903b94793c361b461207b7aa3c133ac/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,251 INFO [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region e903b94793c361b461207b7aa3c133ac 2024-11-13T05:58:25,252 INFO [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e903b94793c361b461207b7aa3c133ac columnFamilyName colfamily31 2024-11-13T05:58:25,252 DEBUG [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,253 INFO [StoreOpener-e903b94793c361b461207b7aa3c133ac-1 {}] regionserver.HStore(327): Store=e903b94793c361b461207b7aa3c133ac/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,253 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for e903b94793c361b461207b7aa3c133ac 2024-11-13T05:58:25,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a65241dbe6c2863f31a0138a73130225, ASSIGN in 410 msec 2024-11-13T05:58:25,254 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T05:58:25,254 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac 2024-11-13T05:58:25,255 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened 98cb12bd0a37bbbfee08e3459a85f735; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68763081, jitterRate=0.02464975416660309}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:58:25,255 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:58:25,255 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac 2024-11-13T05:58:25,255 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for 98cb12bd0a37bbbfee08e3459a85f735: Running coprocessor pre-open hook at 1731477505168Writing region info on filesystem at 1731477505168Initializing all the Stores at 1731477505169 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505169Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505169Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505169Obtaining lock to block concurrent updates at 1731477505186 (+17 ms)Preparing flush snapshotting stores in 98cb12bd0a37bbbfee08e3459a85f735 at 1731477505186Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477505186Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735. at 1731477505186Flushing 98cb12bd0a37bbbfee08e3459a85f735/colfamily11: creating writer at 1731477505186Flushing 98cb12bd0a37bbbfee08e3459a85f735/colfamily11: appending metadata at 1731477505209 (+23 ms)Flushing 98cb12bd0a37bbbfee08e3459a85f735/colfamily11: closing flushed file at 1731477505209Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ddd8d4f: reopening flushed file at 1731477505233 (+24 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 98cb12bd0a37bbbfee08e3459a85f735 in 57ms, sequenceid=4, compaction requested=false; wal=null at 1731477505243 (+10 ms)Cleaning up temporary data from old regions at 1731477505245 (+2 ms)Running coprocessor post-open hooks at 1731477505255 (+10 ms)Region opened successfully at 1731477505255 2024-11-13T05:58:25,256 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac/recovered.edits/0000000000000000004 2024-11-13T05:58:25,256 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735., pid=73, masterSystemTime=1731477505156 2024-11-13T05:58:25,258 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735. 2024-11-13T05:58:25,258 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735. 2024-11-13T05:58:25,258 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a. 2024-11-13T05:58:25,259 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 5ff24ce91fac66115879ac4ec57a346a, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-13T05:58:25,259 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:58:25,259 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=98cb12bd0a37bbbfee08e3459a85f735, regionState=OPEN, openSeqNum=5, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:25,259 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:58:25,259 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:58:25,259 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:58:25,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 98cb12bd0a37bbbfee08e3459a85f735, server=bc386b7bf2fe,40327,1731477314499 because future has completed 2024-11-13T05:58:25,262 INFO [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:58:25,263 INFO [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ff24ce91fac66115879ac4ec57a346a columnFamilyName colfamily11 2024-11-13T05:58:25,263 DEBUG [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,263 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac/recovered.edits/0000000000000000004 2024-11-13T05:58:25,263 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(2902): Flushing e903b94793c361b461207b7aa3c133ac 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:25,264 INFO [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] regionserver.HStore(327): Store=5ff24ce91fac66115879ac4ec57a346a/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,264 INFO [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:58:25,265 INFO [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ff24ce91fac66115879ac4ec57a346a columnFamilyName colfamily21 2024-11-13T05:58:25,265 DEBUG [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=67 2024-11-13T05:58:25,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure 98cb12bd0a37bbbfee08e3459a85f735, server=bc386b7bf2fe,40327,1731477314499 in 260 msec 2024-11-13T05:58:25,267 INFO [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] regionserver.HStore(327): Store=5ff24ce91fac66115879ac4ec57a346a/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,267 INFO [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:58:25,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=98cb12bd0a37bbbfee08e3459a85f735, ASSIGN in 425 msec 2024-11-13T05:58:25,268 INFO [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ff24ce91fac66115879ac4ec57a346a columnFamilyName colfamily31 2024-11-13T05:58:25,268 DEBUG [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,269 INFO [StoreOpener-5ff24ce91fac66115879ac4ec57a346a-1 {}] regionserver.HStore(327): Store=5ff24ce91fac66115879ac4ec57a346a/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,269 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:58:25,270 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:58:25,272 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:58:25,273 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a/recovered.edits/0000000000000000004 2024-11-13T05:58:25,280 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a/recovered.edits/0000000000000000004 2024-11-13T05:58:25,280 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(2902): Flushing 5ff24ce91fac66115879ac4ec57a346a 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:25,289 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac/.tmp/colfamily11/a4b553bfb01a48a8aeaf14aca61a3389 is 52, key is ddd/colfamily11:colfamily11/1731477320578/Put/seqid=0 2024-11-13T05:58:25,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741880_1056 (size=39263) 2024-11-13T05:58:25,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741880_1056 (size=39263) 2024-11-13T05:58:25,299 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a/.tmp/colfamily11/6dcc135acb3149b5b7d8ed30a03fae5c is 52, key is mmm/colfamily11:colfamily11/1731477320711/Put/seqid=0 2024-11-13T05:58:25,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741880_1056 (size=39263) 2024-11-13T05:58:25,301 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac/.tmp/colfamily11/a4b553bfb01a48a8aeaf14aca61a3389 2024-11-13T05:58:25,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741881_1057 (size=39263) 2024-11-13T05:58:25,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741881_1057 (size=39263) 2024-11-13T05:58:25,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741881_1057 (size=39263) 2024-11-13T05:58:25,309 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac/.tmp/colfamily11/a4b553bfb01a48a8aeaf14aca61a3389 as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac/colfamily11/a4b553bfb01a48a8aeaf14aca61a3389 2024-11-13T05:58:25,317 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac/colfamily11/a4b553bfb01a48a8aeaf14aca61a3389, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T05:58:25,318 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for e903b94793c361b461207b7aa3c133ac in 54ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T05:58:25,319 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac/recovered.edits/0000000000000000004 2024-11-13T05:58:25,320 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for e903b94793c361b461207b7aa3c133ac 2024-11-13T05:58:25,320 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for e903b94793c361b461207b7aa3c133ac 2024-11-13T05:58:25,321 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:58:25,323 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for e903b94793c361b461207b7aa3c133ac 2024-11-13T05:58:25,325 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T05:58:25,326 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened e903b94793c361b461207b7aa3c133ac; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58735853, jitterRate=-0.12476758658885956}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:58:25,327 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e903b94793c361b461207b7aa3c133ac 2024-11-13T05:58:25,327 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for e903b94793c361b461207b7aa3c133ac: Running coprocessor pre-open hook at 1731477505244Writing region info on filesystem at 1731477505244Initializing all the Stores at 1731477505245 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505245Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505246 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505246Obtaining lock to block concurrent updates at 1731477505263 (+17 ms)Preparing flush snapshotting stores in e903b94793c361b461207b7aa3c133ac at 1731477505263Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477505264 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac. at 1731477505264Flushing e903b94793c361b461207b7aa3c133ac/colfamily11: creating writer at 1731477505264Flushing e903b94793c361b461207b7aa3c133ac/colfamily11: appending metadata at 1731477505288 (+24 ms)Flushing e903b94793c361b461207b7aa3c133ac/colfamily11: closing flushed file at 1731477505288Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c701bd0: reopening flushed file at 1731477505308 (+20 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for e903b94793c361b461207b7aa3c133ac in 54ms, sequenceid=4, compaction requested=false; wal=null at 1731477505318 (+10 ms)Cleaning up temporary data from old regions at 1731477505320 (+2 ms)Running coprocessor post-open hooks at 1731477505327 (+7 ms)Region opened successfully at 1731477505327 2024-11-13T05:58:25,328 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac., pid=81, masterSystemTime=1731477505161 2024-11-13T05:58:25,330 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac. 2024-11-13T05:58:25,330 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac. 2024-11-13T05:58:25,330 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca. 2024-11-13T05:58:25,330 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 9afb90ba9af49bb3547d4438986e1fca, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-13T05:58:25,330 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=e903b94793c361b461207b7aa3c133ac, regionState=OPEN, openSeqNum=5, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:25,330 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:58:25,331 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:58:25,331 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:58:25,331 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:58:25,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure e903b94793c361b461207b7aa3c133ac, server=bc386b7bf2fe,45383,1731477314448 because future has completed 2024-11-13T05:58:25,333 INFO [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:58:25,334 INFO [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9afb90ba9af49bb3547d4438986e1fca columnFamilyName colfamily11 2024-11-13T05:58:25,334 DEBUG [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,334 INFO [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] regionserver.HStore(327): Store=9afb90ba9af49bb3547d4438986e1fca/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,334 INFO [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:58:25,335 INFO [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9afb90ba9af49bb3547d4438986e1fca columnFamilyName colfamily21 2024-11-13T05:58:25,335 DEBUG [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=68 2024-11-13T05:58:25,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure e903b94793c361b461207b7aa3c133ac, server=bc386b7bf2fe,45383,1731477314448 in 306 msec 2024-11-13T05:58:25,336 INFO [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] regionserver.HStore(327): Store=9afb90ba9af49bb3547d4438986e1fca/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,336 INFO [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:58:25,337 INFO [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9afb90ba9af49bb3547d4438986e1fca columnFamilyName colfamily31 2024-11-13T05:58:25,337 DEBUG [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,338 INFO [StoreOpener-9afb90ba9af49bb3547d4438986e1fca-1 {}] regionserver.HStore(327): Store=9afb90ba9af49bb3547d4438986e1fca/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,338 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:58:25,338 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e903b94793c361b461207b7aa3c133ac, ASSIGN in 495 msec 2024-11-13T05:58:25,339 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:58:25,340 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:58:25,340 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca/recovered.edits/0000000000000000004 2024-11-13T05:58:25,345 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca/recovered.edits/0000000000000000004 2024-11-13T05:58:25,345 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(2902): Flushing 9afb90ba9af49bb3547d4438986e1fca 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:25,361 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca/.tmp/colfamily11/b76ea24aaa1a465bbdf2625b7c8e1cb7 is 52, key is ppp/colfamily11:colfamily11/1731477320735/Put/seqid=0 2024-11-13T05:58:25,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741882_1058 (size=39263) 2024-11-13T05:58:25,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741882_1058 (size=39263) 2024-11-13T05:58:25,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741882_1058 (size=39263) 2024-11-13T05:58:25,368 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca/.tmp/colfamily11/b76ea24aaa1a465bbdf2625b7c8e1cb7 2024-11-13T05:58:25,376 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca/.tmp/colfamily11/b76ea24aaa1a465bbdf2625b7c8e1cb7 as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca/colfamily11/b76ea24aaa1a465bbdf2625b7c8e1cb7 2024-11-13T05:58:25,382 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca/colfamily11/b76ea24aaa1a465bbdf2625b7c8e1cb7, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T05:58:25,382 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 9afb90ba9af49bb3547d4438986e1fca in 37ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T05:58:25,383 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca/recovered.edits/0000000000000000004 2024-11-13T05:58:25,384 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:58:25,384 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:58:25,385 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:58:25,387 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:58:25,390 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T05:58:25,391 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 9afb90ba9af49bb3547d4438986e1fca; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73151427, jitterRate=0.09004120528697968}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:58:25,391 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:58:25,392 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 9afb90ba9af49bb3547d4438986e1fca: Running coprocessor pre-open hook at 1731477505331Writing region info on filesystem at 1731477505331Initializing all the Stores at 1731477505332 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505332Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505333 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505333Obtaining lock to block concurrent updates at 1731477505346 (+13 ms)Preparing flush snapshotting stores in 9afb90ba9af49bb3547d4438986e1fca at 1731477505346Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477505346Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca. at 1731477505346Flushing 9afb90ba9af49bb3547d4438986e1fca/colfamily11: creating writer at 1731477505346Flushing 9afb90ba9af49bb3547d4438986e1fca/colfamily11: appending metadata at 1731477505361 (+15 ms)Flushing 9afb90ba9af49bb3547d4438986e1fca/colfamily11: closing flushed file at 1731477505361Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50c02377: reopening flushed file at 1731477505375 (+14 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 9afb90ba9af49bb3547d4438986e1fca in 37ms, sequenceid=4, compaction requested=false; wal=null at 1731477505382 (+7 ms)Cleaning up temporary data from old regions at 1731477505385 (+3 ms)Running coprocessor post-open hooks at 1731477505391 (+6 ms)Region opened successfully at 1731477505391 2024-11-13T05:58:25,392 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca., pid=75, masterSystemTime=1731477505161 2024-11-13T05:58:25,394 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca. 2024-11-13T05:58:25,395 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca. 2024-11-13T05:58:25,395 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222. 2024-11-13T05:58:25,395 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b613e7fdd93a34284c913bfa5a7b222, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-13T05:58:25,395 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=9afb90ba9af49bb3547d4438986e1fca, regionState=OPEN, openSeqNum=5, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:25,395 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:58:25,395 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:58:25,396 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:58:25,396 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:58:25,398 INFO [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:58:25,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9afb90ba9af49bb3547d4438986e1fca, server=bc386b7bf2fe,45383,1731477314448 because future has completed 2024-11-13T05:58:25,399 INFO [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b613e7fdd93a34284c913bfa5a7b222 columnFamilyName colfamily11 2024-11-13T05:58:25,399 DEBUG [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,400 INFO [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] regionserver.HStore(327): Store=4b613e7fdd93a34284c913bfa5a7b222/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,400 INFO [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:58:25,401 INFO [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b613e7fdd93a34284c913bfa5a7b222 columnFamilyName colfamily21 2024-11-13T05:58:25,401 DEBUG [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=71 2024-11-13T05:58:25,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure 9afb90ba9af49bb3547d4438986e1fca, server=bc386b7bf2fe,45383,1731477314448 in 389 msec 2024-11-13T05:58:25,402 INFO [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] regionserver.HStore(327): Store=4b613e7fdd93a34284c913bfa5a7b222/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,402 INFO [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:58:25,403 INFO [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b613e7fdd93a34284c913bfa5a7b222 columnFamilyName colfamily31 2024-11-13T05:58:25,403 DEBUG [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9afb90ba9af49bb3547d4438986e1fca, ASSIGN in 561 msec 2024-11-13T05:58:25,404 INFO [StoreOpener-4b613e7fdd93a34284c913bfa5a7b222-1 {}] regionserver.HStore(327): Store=4b613e7fdd93a34284c913bfa5a7b222/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,404 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:58:25,405 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:58:25,407 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:58:25,407 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4b613e7fdd93a34284c913bfa5a7b222/recovered.edits/0000000000000000004 2024-11-13T05:58:25,413 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4b613e7fdd93a34284c913bfa5a7b222/recovered.edits/0000000000000000004 2024-11-13T05:58:25,413 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(2902): Flushing 4b613e7fdd93a34284c913bfa5a7b222 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:25,430 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4b613e7fdd93a34284c913bfa5a7b222/.tmp/colfamily11/abdd6dd432ff4ac9ba1ab37666b6db19 is 52, key is jjj/colfamily11:colfamily11/1731477320685/Put/seqid=0 2024-11-13T05:58:25,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741883_1059 (size=39263) 2024-11-13T05:58:25,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741883_1059 (size=39263) 2024-11-13T05:58:25,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741883_1059 (size=39263) 2024-11-13T05:58:25,438 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4b613e7fdd93a34284c913bfa5a7b222/.tmp/colfamily11/abdd6dd432ff4ac9ba1ab37666b6db19 2024-11-13T05:58:25,446 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4b613e7fdd93a34284c913bfa5a7b222/.tmp/colfamily11/abdd6dd432ff4ac9ba1ab37666b6db19 as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4b613e7fdd93a34284c913bfa5a7b222/colfamily11/abdd6dd432ff4ac9ba1ab37666b6db19 2024-11-13T05:58:25,454 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4b613e7fdd93a34284c913bfa5a7b222/colfamily11/abdd6dd432ff4ac9ba1ab37666b6db19, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T05:58:25,454 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 4b613e7fdd93a34284c913bfa5a7b222 in 41ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T05:58:25,455 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4b613e7fdd93a34284c913bfa5a7b222/recovered.edits/0000000000000000004 2024-11-13T05:58:25,456 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:58:25,456 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:58:25,457 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:58:25,459 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:58:25,461 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4b613e7fdd93a34284c913bfa5a7b222/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T05:58:25,462 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 4b613e7fdd93a34284c913bfa5a7b222; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61115560, jitterRate=-0.08930718898773193}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:58:25,462 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:58:25,462 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 4b613e7fdd93a34284c913bfa5a7b222: Running coprocessor pre-open hook at 1731477505396Writing region info on filesystem at 1731477505396Initializing all the Stores at 1731477505397 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505397Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505397Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505397Obtaining lock to block concurrent updates at 1731477505413 (+16 ms)Preparing flush snapshotting stores in 4b613e7fdd93a34284c913bfa5a7b222 at 1731477505413Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477505414 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222. at 1731477505414Flushing 4b613e7fdd93a34284c913bfa5a7b222/colfamily11: creating writer at 1731477505414Flushing 4b613e7fdd93a34284c913bfa5a7b222/colfamily11: appending metadata at 1731477505430 (+16 ms)Flushing 4b613e7fdd93a34284c913bfa5a7b222/colfamily11: closing flushed file at 1731477505430Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@595c38b9: reopening flushed file at 1731477505445 (+15 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 4b613e7fdd93a34284c913bfa5a7b222 in 41ms, sequenceid=4, compaction requested=false; wal=null at 1731477505454 (+9 ms)Cleaning up temporary data from old regions at 1731477505456 (+2 ms)Running coprocessor post-open hooks at 1731477505462 (+6 ms)Region opened successfully at 1731477505462 2024-11-13T05:58:25,463 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222., pid=80, masterSystemTime=1731477505161 2024-11-13T05:58:25,465 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222. 2024-11-13T05:58:25,465 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222. 2024-11-13T05:58:25,466 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=4b613e7fdd93a34284c913bfa5a7b222, regionState=OPEN, openSeqNum=5, regionLocation=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:25,468 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b613e7fdd93a34284c913bfa5a7b222, server=bc386b7bf2fe,45383,1731477314448 because future has completed 2024-11-13T05:58:25,470 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=65 2024-11-13T05:58:25,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure 4b613e7fdd93a34284c913bfa5a7b222, server=bc386b7bf2fe,45383,1731477314448 in 445 msec 2024-11-13T05:58:25,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4b613e7fdd93a34284c913bfa5a7b222, ASSIGN in 630 msec 2024-11-13T05:58:25,710 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a/.tmp/colfamily11/6dcc135acb3149b5b7d8ed30a03fae5c 2024-11-13T05:58:25,717 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a/.tmp/colfamily11/6dcc135acb3149b5b7d8ed30a03fae5c as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a/colfamily11/6dcc135acb3149b5b7d8ed30a03fae5c 2024-11-13T05:58:25,725 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a/colfamily11/6dcc135acb3149b5b7d8ed30a03fae5c, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T05:58:25,725 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5ff24ce91fac66115879ac4ec57a346a in 445ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T05:58:25,726 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a/recovered.edits/0000000000000000004 2024-11-13T05:58:25,728 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:58:25,728 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:58:25,728 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:58:25,730 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:58:25,733 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T05:58:25,734 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 5ff24ce91fac66115879ac4ec57a346a; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73241799, jitterRate=0.09138785302639008}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:58:25,734 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:58:25,734 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 5ff24ce91fac66115879ac4ec57a346a: Running coprocessor pre-open hook at 1731477505259Writing region info on filesystem at 1731477505259Initializing all the Stores at 1731477505261 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505261Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505262 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505262Obtaining lock to block concurrent updates at 1731477505280 (+18 ms)Preparing flush snapshotting stores in 5ff24ce91fac66115879ac4ec57a346a at 1731477505280Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477505280Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a. at 1731477505280Flushing 5ff24ce91fac66115879ac4ec57a346a/colfamily11: creating writer at 1731477505280Flushing 5ff24ce91fac66115879ac4ec57a346a/colfamily11: appending metadata at 1731477505299 (+19 ms)Flushing 5ff24ce91fac66115879ac4ec57a346a/colfamily11: closing flushed file at 1731477505299Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6add5a1f: reopening flushed file at 1731477505716 (+417 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5ff24ce91fac66115879ac4ec57a346a in 445ms, sequenceid=4, compaction requested=false; wal=null at 1731477505725 (+9 ms)Cleaning up temporary data from old regions at 1731477505728 (+3 ms)Running coprocessor post-open hooks at 1731477505734 (+6 ms)Region opened successfully at 1731477505734 2024-11-13T05:58:25,735 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a., pid=78, masterSystemTime=1731477505156 2024-11-13T05:58:25,737 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a. 2024-11-13T05:58:25,737 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a. 2024-11-13T05:58:25,737 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7. 2024-11-13T05:58:25,737 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 82c0a84200b3abe2cb346258143147c7, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-13T05:58:25,737 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=5ff24ce91fac66115879ac4ec57a346a, regionState=OPEN, openSeqNum=5, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:25,737 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:58:25,738 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:58:25,738 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:58:25,738 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:58:25,739 INFO [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:58:25,739 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ff24ce91fac66115879ac4ec57a346a, server=bc386b7bf2fe,40327,1731477314499 because future has completed 2024-11-13T05:58:25,740 INFO [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82c0a84200b3abe2cb346258143147c7 columnFamilyName colfamily11 2024-11-13T05:58:25,740 DEBUG [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,741 INFO [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] regionserver.HStore(327): Store=82c0a84200b3abe2cb346258143147c7/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,741 INFO [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:58:25,742 INFO [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82c0a84200b3abe2cb346258143147c7 columnFamilyName colfamily21 2024-11-13T05:58:25,742 DEBUG [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=69 2024-11-13T05:58:25,742 INFO [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] regionserver.HStore(327): Store=82c0a84200b3abe2cb346258143147c7/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 5ff24ce91fac66115879ac4ec57a346a, server=bc386b7bf2fe,40327,1731477314499 in 723 msec 2024-11-13T05:58:25,743 INFO [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:58:25,743 INFO [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82c0a84200b3abe2cb346258143147c7 columnFamilyName colfamily31 2024-11-13T05:58:25,744 DEBUG [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,744 INFO [StoreOpener-82c0a84200b3abe2cb346258143147c7-1 {}] regionserver.HStore(327): Store=82c0a84200b3abe2cb346258143147c7/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,744 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:58:25,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ff24ce91fac66115879ac4ec57a346a, ASSIGN in 902 msec 2024-11-13T05:58:25,745 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7 2024-11-13T05:58:25,746 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7 2024-11-13T05:58:25,747 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7/recovered.edits/0000000000000000004 2024-11-13T05:58:25,752 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7/recovered.edits/0000000000000000004 2024-11-13T05:58:25,752 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(2902): Flushing 82c0a84200b3abe2cb346258143147c7 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:25,773 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7/.tmp/colfamily11/a1232e743c994907ae769574eaeca512 is 52, key is sss/colfamily11:colfamily11/1731477320760/Put/seqid=0 2024-11-13T05:58:25,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741884_1060 (size=39263) 2024-11-13T05:58:25,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741884_1060 (size=39263) 2024-11-13T05:58:25,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741884_1060 (size=39263) 2024-11-13T05:58:25,780 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7/.tmp/colfamily11/a1232e743c994907ae769574eaeca512 2024-11-13T05:58:25,787 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7/.tmp/colfamily11/a1232e743c994907ae769574eaeca512 as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7/colfamily11/a1232e743c994907ae769574eaeca512 2024-11-13T05:58:25,795 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7/colfamily11/a1232e743c994907ae769574eaeca512, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T05:58:25,796 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 82c0a84200b3abe2cb346258143147c7 in 44ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T05:58:25,797 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7/recovered.edits/0000000000000000004 2024-11-13T05:58:25,798 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:58:25,798 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:58:25,799 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:58:25,801 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:58:25,803 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T05:58:25,804 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 82c0a84200b3abe2cb346258143147c7; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61168107, jitterRate=-0.08852417767047882}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:58:25,804 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:58:25,804 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 82c0a84200b3abe2cb346258143147c7: Running coprocessor pre-open hook at 1731477505738Writing region info on filesystem at 1731477505738Initializing all the Stores at 1731477505739 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505739Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505739Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505739Obtaining lock to block concurrent updates at 1731477505752 (+13 ms)Preparing flush snapshotting stores in 82c0a84200b3abe2cb346258143147c7 at 1731477505752Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477505752Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7. at 1731477505752Flushing 82c0a84200b3abe2cb346258143147c7/colfamily11: creating writer at 1731477505752Flushing 82c0a84200b3abe2cb346258143147c7/colfamily11: appending metadata at 1731477505772 (+20 ms)Flushing 82c0a84200b3abe2cb346258143147c7/colfamily11: closing flushed file at 1731477505772Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ffdded5: reopening flushed file at 1731477505786 (+14 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 82c0a84200b3abe2cb346258143147c7 in 44ms, sequenceid=4, compaction requested=false; wal=null at 1731477505796 (+10 ms)Cleaning up temporary data from old regions at 1731477505798 (+2 ms)Running coprocessor post-open hooks at 1731477505804 (+6 ms)Region opened successfully at 1731477505804 2024-11-13T05:58:25,805 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7., pid=79, masterSystemTime=1731477505156 2024-11-13T05:58:25,807 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7. 2024-11-13T05:58:25,807 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7. 2024-11-13T05:58:25,807 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d. 2024-11-13T05:58:25,807 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 84316f2c5b758522338a907d7e8f386d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-13T05:58:25,807 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=82c0a84200b3abe2cb346258143147c7, regionState=OPEN, openSeqNum=5, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:25,808 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:58:25,808 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:58:25,808 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:58:25,808 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:58:25,809 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82c0a84200b3abe2cb346258143147c7, server=bc386b7bf2fe,40327,1731477314499 because future has completed 2024-11-13T05:58:25,809 INFO [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:58:25,810 INFO [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 84316f2c5b758522338a907d7e8f386d columnFamilyName colfamily11 2024-11-13T05:58:25,810 DEBUG [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,811 INFO [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] regionserver.HStore(327): Store=84316f2c5b758522338a907d7e8f386d/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,811 INFO [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:58:25,812 INFO [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 84316f2c5b758522338a907d7e8f386d columnFamilyName colfamily21 2024-11-13T05:58:25,812 DEBUG [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,813 INFO [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] regionserver.HStore(327): Store=84316f2c5b758522338a907d7e8f386d/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=72 2024-11-13T05:58:25,813 INFO [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:58:25,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure 82c0a84200b3abe2cb346258143147c7, server=bc386b7bf2fe,40327,1731477314499 in 790 msec 2024-11-13T05:58:25,814 INFO [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 84316f2c5b758522338a907d7e8f386d columnFamilyName colfamily31 2024-11-13T05:58:25,814 DEBUG [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,814 INFO [StoreOpener-84316f2c5b758522338a907d7e8f386d-1 {}] regionserver.HStore(327): Store=84316f2c5b758522338a907d7e8f386d/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,814 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:58:25,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=82c0a84200b3abe2cb346258143147c7, ASSIGN in 972 msec 2024-11-13T05:58:25,815 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d 2024-11-13T05:58:25,816 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d 2024-11-13T05:58:25,817 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d/recovered.edits/0000000000000000004 2024-11-13T05:58:25,822 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d/recovered.edits/0000000000000000004 2024-11-13T05:58:25,822 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(2902): Flushing 84316f2c5b758522338a907d7e8f386d 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:25,838 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d/.tmp/colfamily11/3db6742d14e74636a8c261699c4b7c1c is 52, key is aaa/colfamily11:colfamily11/1731477320537/Put/seqid=0 2024-11-13T05:58:25,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741885_1061 (size=39263) 2024-11-13T05:58:25,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741885_1061 (size=39263) 2024-11-13T05:58:25,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741885_1061 (size=39263) 2024-11-13T05:58:25,845 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d/.tmp/colfamily11/3db6742d14e74636a8c261699c4b7c1c 2024-11-13T05:58:25,855 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d/.tmp/colfamily11/3db6742d14e74636a8c261699c4b7c1c as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d/colfamily11/3db6742d14e74636a8c261699c4b7c1c 2024-11-13T05:58:25,862 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d/colfamily11/3db6742d14e74636a8c261699c4b7c1c, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T05:58:25,862 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 84316f2c5b758522338a907d7e8f386d in 40ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T05:58:25,863 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d/recovered.edits/0000000000000000004 2024-11-13T05:58:25,864 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:58:25,864 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:58:25,865 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:58:25,866 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:58:25,869 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T05:58:25,870 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened 84316f2c5b758522338a907d7e8f386d; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71273675, jitterRate=0.06206052005290985}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:58:25,870 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:58:25,870 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 84316f2c5b758522338a907d7e8f386d: Running coprocessor pre-open hook at 1731477505808Writing region info on filesystem at 1731477505808Initializing all the Stores at 1731477505809 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505809Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505809Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505809Obtaining lock to block concurrent updates at 1731477505822 (+13 ms)Preparing flush snapshotting stores in 84316f2c5b758522338a907d7e8f386d at 1731477505822Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477505822Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d. at 1731477505822Flushing 84316f2c5b758522338a907d7e8f386d/colfamily11: creating writer at 1731477505822Flushing 84316f2c5b758522338a907d7e8f386d/colfamily11: appending metadata at 1731477505838 (+16 ms)Flushing 84316f2c5b758522338a907d7e8f386d/colfamily11: closing flushed file at 1731477505838Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cf2be85: reopening flushed file at 1731477505854 (+16 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 84316f2c5b758522338a907d7e8f386d in 40ms, sequenceid=4, compaction requested=false; wal=null at 1731477505862 (+8 ms)Cleaning up temporary data from old regions at 1731477505864 (+2 ms)Running coprocessor post-open hooks at 1731477505870 (+6 ms)Region opened successfully at 1731477505870 2024-11-13T05:58:25,871 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d., pid=74, masterSystemTime=1731477505156 2024-11-13T05:58:25,873 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d. 2024-11-13T05:58:25,873 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d. 2024-11-13T05:58:25,873 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1. 2024-11-13T05:58:25,873 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 11abc6d6caf592b351052ae8adf50fd1, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-13T05:58:25,873 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=84316f2c5b758522338a907d7e8f386d, regionState=OPEN, openSeqNum=5, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:25,874 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:58:25,874 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:58:25,874 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:58:25,874 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:58:25,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 84316f2c5b758522338a907d7e8f386d, server=bc386b7bf2fe,40327,1731477314499 because future has completed 2024-11-13T05:58:25,875 INFO [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:58:25,876 INFO [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11abc6d6caf592b351052ae8adf50fd1 columnFamilyName colfamily11 2024-11-13T05:58:25,876 DEBUG [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,877 INFO [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] regionserver.HStore(327): Store=11abc6d6caf592b351052ae8adf50fd1/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,877 INFO [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:58:25,878 INFO [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11abc6d6caf592b351052ae8adf50fd1 columnFamilyName colfamily21 2024-11-13T05:58:25,878 DEBUG [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=64 2024-11-13T05:58:25,879 INFO [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] regionserver.HStore(327): Store=11abc6d6caf592b351052ae8adf50fd1/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 84316f2c5b758522338a907d7e8f386d, server=bc386b7bf2fe,40327,1731477314499 in 869 msec 2024-11-13T05:58:25,879 INFO [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:58:25,880 INFO [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11abc6d6caf592b351052ae8adf50fd1 columnFamilyName colfamily31 2024-11-13T05:58:25,880 DEBUG [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:58:25,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=84316f2c5b758522338a907d7e8f386d, ASSIGN in 1.0380 sec 2024-11-13T05:58:25,880 INFO [StoreOpener-11abc6d6caf592b351052ae8adf50fd1-1 {}] regionserver.HStore(327): Store=11abc6d6caf592b351052ae8adf50fd1/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:58:25,881 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:58:25,881 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:58:25,883 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:58:25,883 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1/recovered.edits/0000000000000000004 2024-11-13T05:58:25,888 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1/recovered.edits/0000000000000000004 2024-11-13T05:58:25,888 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(2902): Flushing 11abc6d6caf592b351052ae8adf50fd1 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:25,904 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1/.tmp/colfamily11/8faba3afe8734acb936444e5684919b1 is 52, key is ggg/colfamily11:colfamily11/1731477320640/Put/seqid=0 2024-11-13T05:58:25,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741886_1062 (size=39263) 2024-11-13T05:58:25,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741886_1062 (size=39263) 2024-11-13T05:58:25,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741886_1062 (size=39263) 2024-11-13T05:58:25,911 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1/.tmp/colfamily11/8faba3afe8734acb936444e5684919b1 2024-11-13T05:58:25,918 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1/.tmp/colfamily11/8faba3afe8734acb936444e5684919b1 as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1/colfamily11/8faba3afe8734acb936444e5684919b1 2024-11-13T05:58:25,924 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1/colfamily11/8faba3afe8734acb936444e5684919b1, entries=703, sequenceid=4, filesize=38.3 K 2024-11-13T05:58:25,924 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 11abc6d6caf592b351052ae8adf50fd1 in 36ms, sequenceid=4, compaction requested=false; wal=null 2024-11-13T05:58:25,925 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1/recovered.edits/0000000000000000004 2024-11-13T05:58:25,926 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:58:25,926 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:58:25,926 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T05:58:25,928 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:58:25,930 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T05:58:25,931 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 11abc6d6caf592b351052ae8adf50fd1; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72636358, jitterRate=0.08236607909202576}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T05:58:25,931 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:58:25,931 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 11abc6d6caf592b351052ae8adf50fd1: Running coprocessor pre-open hook at 1731477505874Writing region info on filesystem at 1731477505874Initializing all the Stores at 1731477505875 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505875Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505875Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477505875Obtaining lock to block concurrent updates at 1731477505888 (+13 ms)Preparing flush snapshotting stores in 11abc6d6caf592b351052ae8adf50fd1 at 1731477505888Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477505888Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1. at 1731477505888Flushing 11abc6d6caf592b351052ae8adf50fd1/colfamily11: creating writer at 1731477505888Flushing 11abc6d6caf592b351052ae8adf50fd1/colfamily11: appending metadata at 1731477505904 (+16 ms)Flushing 11abc6d6caf592b351052ae8adf50fd1/colfamily11: closing flushed file at 1731477505904Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23bb20a6: reopening flushed file at 1731477505917 (+13 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 11abc6d6caf592b351052ae8adf50fd1 in 36ms, sequenceid=4, compaction requested=false; wal=null at 1731477505924 (+7 ms)Cleaning up temporary data from old regions at 1731477505926 (+2 ms)Running coprocessor post-open hooks at 1731477505931 (+5 ms)Region opened successfully at 1731477505931 2024-11-13T05:58:25,932 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1., pid=77, masterSystemTime=1731477505156 2024-11-13T05:58:25,934 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1. 2024-11-13T05:58:25,934 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1. 2024-11-13T05:58:25,934 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=11abc6d6caf592b351052ae8adf50fd1, regionState=OPEN, openSeqNum=5, regionLocation=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:25,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11abc6d6caf592b351052ae8adf50fd1, server=bc386b7bf2fe,40327,1731477314499 because future has completed 2024-11-13T05:58:25,939 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=66 2024-11-13T05:58:25,939 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 11abc6d6caf592b351052ae8adf50fd1, server=bc386b7bf2fe,40327,1731477314499 in 922 msec 2024-11-13T05:58:25,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=57 2024-11-13T05:58:25,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=11abc6d6caf592b351052ae8adf50fd1, ASSIGN in 1.0980 sec 2024-11-13T05:58:25,942 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server bc386b7bf2fe,38823,1731477314332 after splitting done 2024-11-13T05:58:25,942 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-13T05:58:25,942 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:58:25,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:58:25,942 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:58:26,091 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 8 exec state=pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:58:26,091 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T05:58:26,091 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:58:26,091 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T05:58:26,092 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T05:58:26,092 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T05:58:26,092 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T05:58:26,092 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T05:58:26,093 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T05:58:26,094 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T05:58:26,095 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:58:26,101 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T05:58:26,101 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T05:58:26,101 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T05:58:26,101 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T05:58:26,101 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T05:58:26,101 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T05:58:26,101 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T05:58:26,101 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T05:58:26,101 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T05:58:26,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T05:58:26,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T05:58:26,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T05:58:26,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T05:58:26,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T05:58:26,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T05:58:26,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T05:58:26,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T05:58:26,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T05:58:26,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T05:58:26,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T05:58:26,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T05:58:26,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T05:58:26,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T05:58:26,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T05:58:26,102 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T05:58:26,103 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T05:58:26,103 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-13T05:58:26,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_FINISH, hasLock=false; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true 2024-11-13T05:58:26,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:26,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:26,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:58:26,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:26,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.1731477316011, worker=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:26,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.1731477316011, worker=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:26,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=84316f2c5b758522338a907d7e8f386d, ASSIGN 2024-11-13T05:58:26,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4b613e7fdd93a34284c913bfa5a7b222, ASSIGN 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=11abc6d6caf592b351052ae8adf50fd1, ASSIGN 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=98cb12bd0a37bbbfee08e3459a85f735, ASSIGN 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e903b94793c361b461207b7aa3c133ac, ASSIGN 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ff24ce91fac66115879ac4ec57a346a, ASSIGN 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a65241dbe6c2863f31a0138a73130225, ASSIGN 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9afb90ba9af49bb3547d4438986e1fca, ASSIGN 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=82c0a84200b3abe2cb346258143147c7, ASSIGN 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure 98cb12bd0a37bbbfee08e3459a85f735, server=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 84316f2c5b758522338a907d7e8f386d, server=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure 9afb90ba9af49bb3547d4438986e1fca, server=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure a65241dbe6c2863f31a0138a73130225, server=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 11abc6d6caf592b351052ae8adf50fd1, server=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 5ff24ce91fac66115879ac4ec57a346a, server=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure 82c0a84200b3abe2cb346258143147c7, server=bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure 4b613e7fdd93a34284c913bfa5a7b222, server=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure e903b94793c361b461207b7aa3c133ac, server=bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_FINISH, hasLock=false; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.meta.1731477316432.meta, worker=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure bc386b7bf2fe%2C38823%2C1731477314332.1731477316011, worker=bc386b7bf2fe,40327,1731477314499 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure bc386b7bf2fe%2C38823%2C1731477314332.1731477316011, worker=bc386b7bf2fe,40327,1731477314499 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=84316f2c5b758522338a907d7e8f386d, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4b613e7fdd93a34284c913bfa5a7b222, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=11abc6d6caf592b351052ae8adf50fd1, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=98cb12bd0a37bbbfee08e3459a85f735, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e903b94793c361b461207b7aa3c133ac, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ff24ce91fac66115879ac4ec57a346a, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a65241dbe6c2863f31a0138a73130225, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9afb90ba9af49bb3547d4438986e1fca, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=82c0a84200b3abe2cb346258143147c7, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure 98cb12bd0a37bbbfee08e3459a85f735, server=bc386b7bf2fe,40327,1731477314499 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 84316f2c5b758522338a907d7e8f386d, server=bc386b7bf2fe,40327,1731477314499 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure 9afb90ba9af49bb3547d4438986e1fca, server=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure a65241dbe6c2863f31a0138a73130225, server=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 11abc6d6caf592b351052ae8adf50fd1, server=bc386b7bf2fe,40327,1731477314499 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 5ff24ce91fac66115879ac4ec57a346a, server=bc386b7bf2fe,40327,1731477314499 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure 82c0a84200b3abe2cb346258143147c7, server=bc386b7bf2fe,40327,1731477314499 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure 4b613e7fdd93a34284c913bfa5a7b222, server=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure e903b94793c361b461207b7aa3c133ac, server=bc386b7bf2fe,45383,1731477314448 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T05:58:26,107 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-13T05:58:26,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=61 2024-11-13T05:58:26,107 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=bc386b7bf2fe,45383,1731477314448, table=hbase:meta, region=1588230740 2024-11-13T05:58:26,107 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,45383,1731477314448, state=OPEN 2024-11-13T05:58:26,159 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:58:26,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:26,164 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:26,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:58:26,164 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:26,164 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:26,164 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:58:26,165 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731477536165 2024-11-13T05:58:26,165 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T05:58:26,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=84316f2c5b758522338a907d7e8f386d, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=5 2024-11-13T05:58:26,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=200f36be3ca8364314512c33e4ae22d6, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:26,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a840e8f158b2eaf5c08595bd38db426e, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:26,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e903b94793c361b461207b7aa3c133ac, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=5 2024-11-13T05:58:26,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=19321613b69704e42207868d450f785b, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:26,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1a492bcf61cc6ef0604137033fc7c90b, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:26,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=11abc6d6caf592b351052ae8adf50fd1, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=5 2024-11-13T05:58:26,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=408b59e776ffab17a3ee0e8fb465c67c, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:26,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=89e8c9d6d5af67c75803894f4aea8ecc, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:26,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4b613e7fdd93a34284c913bfa5a7b222, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=5 2024-11-13T05:58:26,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b540ac2e8c4575b424d481848c4b4e52, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:26,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10c268d6da33b0a4590a478c0f32fd70, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:26,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ff24ce91fac66115879ac4ec57a346a, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=5 2024-11-13T05:58:26,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f7c7452ce5410ec8dcf456adb9b7db6b, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:26,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=43fd53cccd0bd76519c89a78d6bf6131, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:26,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9afb90ba9af49bb3547d4438986e1fca, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=5 2024-11-13T05:58:26,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=408dfc042fd5f2d964a15b01fd44966c, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:26,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bd21adcd80ce6ee8c5ed1a581c04749, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:26,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=82c0a84200b3abe2cb346258143147c7, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=5 2024-11-13T05:58:26,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2810327548325125e764254e87017046, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:26,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e86ecaa5f83edede0f153db264078a94, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:26,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a65241dbe6c2863f31a0138a73130225, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=5 2024-11-13T05:58:26,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff62c0b5be01546e2b1563e065750a3b, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:26,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=80710108d60e6f4a1b8975ccfe51e50e, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=2 2024-11-13T05:58:26,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=98cb12bd0a37bbbfee08e3459a85f735, regionState=OPEN, lastHost=bc386b7bf2fe,40327,1731477314499, regionLocation=bc386b7bf2fe,40327,1731477314499, openSeqNum=5 2024-11-13T05:58:26,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90b03edd8043eabbad2ddbd32409c478, regionState=OPEN, lastHost=bc386b7bf2fe,45383,1731477314448, regionLocation=bc386b7bf2fe,45383,1731477314448, openSeqNum=2 2024-11-13T05:58:26,179 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T05:58:26,179 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731477566179 2024-11-13T05:58:26,179 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731477626179 2024-11-13T05:58:26,179 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-13T05:58:26,179 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server bc386b7bf2fe,38823,1731477314332 after splitting done 2024-11-13T05:58:26,179 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-13T05:58:26,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; ServerCrashProcedure bc386b7bf2fe,38823,1731477314332, splitWal=true, meta=true in 3 mins, 4.556 sec 2024-11-13T05:58:26,433 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45448, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:58:26,436 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:38823 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: bc386b7bf2fe/172.17.0.2:38823 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:58:26,437 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2, error=java.net.ConnectException: Call to address=bc386b7bf2fe:38823 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:58:26,437 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 is java.net.ConnectException: Connection refused 2024-11-13T05:58:26,437 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 from cache 2024-11-13T05:58:26,437 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:38823 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: bc386b7bf2fe/172.17.0.2:38823 2024-11-13T05:58:26,548 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:38823 this server is in the failed servers list 2024-11-13T05:58:26,549 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:58:26,549 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:58:26,549 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,38823,1731477314332, seqNum=-1 from cache 2024-11-13T05:58:26,658 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:58:26,661 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45450, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:58:26,663 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45383,1731477314448, seqNum=-1] 2024-11-13T05:58:26,666 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:58:26,671 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=5] 2024-11-13T05:58:26,673 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:58:26,695 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:38823 this server is in the failed servers list 2024-11-13T05:58:26,696 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:58:26,696 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:58:26,696 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 from cache 2024-11-13T05:58:26,810 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=5] 2024-11-13T05:58:26,828 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:38823 this server is in the failed servers list 2024-11-13T05:58:26,829 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:58:26,829 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:58:26,829 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 from cache 2024-11-13T05:58:26,942 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=5] 2024-11-13T05:58:26,958 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:38823 this server is in the failed servers list 2024-11-13T05:58:26,959 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:58:26,959 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:58:26,959 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 from cache 2024-11-13T05:58:27,072 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=5] 2024-11-13T05:58:27,086 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:38823 this server is in the failed servers list 2024-11-13T05:58:27,087 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:58:27,087 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:58:27,087 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 from cache 2024-11-13T05:58:27,203 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=5] 2024-11-13T05:58:27,216 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:38823 this server is in the failed servers list 2024-11-13T05:58:27,216 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:58:27,216 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:58:27,216 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 from cache 2024-11-13T05:58:27,333 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=5] 2024-11-13T05:58:27,345 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:38823 this server is in the failed servers list 2024-11-13T05:58:27,345 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:58:27,345 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:58:27,345 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 from cache 2024-11-13T05:58:27,453 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sss', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=5] 2024-11-13T05:58:27,466 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:38823 this server is in the failed servers list 2024-11-13T05:58:27,467 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:58:27,467 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:58:27,467 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 from cache 2024-11-13T05:58:27,584 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225., hostname=bc386b7bf2fe,45383,1731477314448, seqNum=5] 2024-11-13T05:58:27,595 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to bc386b7bf2fe:38823 this server is in the failed servers list 2024-11-13T05:58:27,595 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=bc386b7bf2fe:38823 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:58:27,595 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: bc386b7bf2fe:38823 2024-11-13T05:58:27,595 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735., hostname=bc386b7bf2fe,38823,1731477314332, seqNum=2 from cache 2024-11-13T05:58:27,706 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735., hostname=bc386b7bf2fe,40327,1731477314499, seqNum=5] 2024-11-13T05:58:27,768 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-13T05:58:27,768 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T05:58:27,768 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:58:27,768 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSCPBase.tearDown(TestSCPBase.java:72) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:58:27,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:58:27,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:58:27,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:58:27,769 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T05:58:27,769 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1666420094, stopped=false 2024-11-13T05:58:27,769 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=bc386b7bf2fe,37807,1731477313603 2024-11-13T05:58:27,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:58:27,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:58:27,780 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:58:27,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:58:27,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:58:27,780 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:58:27,780 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:58:27,780 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:58:27,781 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSCPBase.tearDown(TestSCPBase.java:72) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:58:27,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:58:27,781 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:58:27,781 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:58:27,781 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:58:27,781 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bc386b7bf2fe,45383,1731477314448' ***** 2024-11-13T05:58:27,782 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T05:58:27,782 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bc386b7bf2fe,40327,1731477314499' ***** 2024-11-13T05:58:27,782 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T05:58:27,782 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T05:58:27,782 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T05:58:27,782 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T05:58:27,782 INFO [RS:1;bc386b7bf2fe:45383 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T05:58:27,782 INFO [regionserver/bc386b7bf2fe:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:58:27,782 INFO [RS:1;bc386b7bf2fe:45383 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T05:58:27,783 INFO [regionserver/bc386b7bf2fe:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:58:27,783 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(3091): Received CLOSE for 90b03edd8043eabbad2ddbd32409c478 2024-11-13T05:58:27,783 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T05:58:27,783 INFO [RS:2;bc386b7bf2fe:40327 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T05:58:27,783 INFO [RS:2;bc386b7bf2fe:40327 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T05:58:27,783 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(3091): Received CLOSE for a840e8f158b2eaf5c08595bd38db426e 2024-11-13T05:58:27,783 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(3091): Received CLOSE for 43fd53cccd0bd76519c89a78d6bf6131 2024-11-13T05:58:27,783 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(3091): Received CLOSE for 89e8c9d6d5af67c75803894f4aea8ecc 2024-11-13T05:58:27,783 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(3091): Received CLOSE for 9bd21adcd80ce6ee8c5ed1a581c04749 2024-11-13T05:58:27,784 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(3091): Received CLOSE for 1a492bcf61cc6ef0604137033fc7c90b 2024-11-13T05:58:27,784 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(3091): Received CLOSE for 5ff24ce91fac66115879ac4ec57a346a 2024-11-13T05:58:27,784 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(3091): Received CLOSE for 84316f2c5b758522338a907d7e8f386d 2024-11-13T05:58:27,784 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(3091): Received CLOSE for e86ecaa5f83edede0f153db264078a94 2024-11-13T05:58:27,784 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(3091): Received CLOSE for a65241dbe6c2863f31a0138a73130225 2024-11-13T05:58:27,784 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(3091): Received CLOSE for 98cb12bd0a37bbbfee08e3459a85f735 2024-11-13T05:58:27,784 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(3091): Received CLOSE for 200f36be3ca8364314512c33e4ae22d6 2024-11-13T05:58:27,784 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(3091): Received CLOSE for 80710108d60e6f4a1b8975ccfe51e50e 2024-11-13T05:58:27,784 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(3091): Received CLOSE for ff62c0b5be01546e2b1563e065750a3b 2024-11-13T05:58:27,784 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 90b03edd8043eabbad2ddbd32409c478, disabling compactions & flushes 2024-11-13T05:58:27,784 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a840e8f158b2eaf5c08595bd38db426e, disabling compactions & flushes 2024-11-13T05:58:27,784 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(3091): Received CLOSE for 4b613e7fdd93a34284c913bfa5a7b222 2024-11-13T05:58:27,784 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(3091): Received CLOSE for 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:58:27,784 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478. 2024-11-13T05:58:27,784 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e. 2024-11-13T05:58:27,784 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(3091): Received CLOSE for 19321613b69704e42207868d450f785b 2024-11-13T05:58:27,784 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(3091): Received CLOSE for 11abc6d6caf592b351052ae8adf50fd1 2024-11-13T05:58:27,784 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478. 2024-11-13T05:58:27,784 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e. 2024-11-13T05:58:27,784 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(3091): Received CLOSE for 9afb90ba9af49bb3547d4438986e1fca 2024-11-13T05:58:27,784 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478. after waiting 0 ms 2024-11-13T05:58:27,784 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(3091): Received CLOSE for 10c268d6da33b0a4590a478c0f32fd70 2024-11-13T05:58:27,784 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e. after waiting 0 ms 2024-11-13T05:58:27,784 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(3091): Received CLOSE for f7c7452ce5410ec8dcf456adb9b7db6b 2024-11-13T05:58:27,784 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478. 2024-11-13T05:58:27,784 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e. 2024-11-13T05:58:27,784 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(3091): Received CLOSE for e903b94793c361b461207b7aa3c133ac 2024-11-13T05:58:27,784 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(3091): Received CLOSE for 2810327548325125e764254e87017046 2024-11-13T05:58:27,784 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 90b03edd8043eabbad2ddbd32409c478 3/3 column families, dataSize=48 B heapSize=888 B 2024-11-13T05:58:27,784 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(959): stopping server bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:27,784 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(3091): Received CLOSE for 408dfc042fd5f2d964a15b01fd44966c 2024-11-13T05:58:27,784 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(3091): Received CLOSE for 408b59e776ffab17a3ee0e8fb465c67c 2024-11-13T05:58:27,784 INFO [RS:2;bc386b7bf2fe:40327 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:58:27,784 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(3091): Received CLOSE for b540ac2e8c4575b424d481848c4b4e52 2024-11-13T05:58:27,784 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a840e8f158b2eaf5c08595bd38db426e 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:27,784 INFO [RS:2;bc386b7bf2fe:40327 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;bc386b7bf2fe:40327. 2024-11-13T05:58:27,784 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(959): stopping server bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:27,785 INFO [RS:1;bc386b7bf2fe:45383 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:58:27,785 DEBUG [RS:2;bc386b7bf2fe:40327 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:58:27,785 INFO [RS:1;bc386b7bf2fe:45383 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;bc386b7bf2fe:45383. 2024-11-13T05:58:27,785 DEBUG [RS:2;bc386b7bf2fe:40327 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:58:27,785 DEBUG [RS:1;bc386b7bf2fe:45383 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:58:27,785 DEBUG [RS:1;bc386b7bf2fe:45383 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:58:27,785 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-13T05:58:27,785 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T05:58:27,785 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T05:58:27,785 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T05:58:27,785 DEBUG [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(1351): Waiting on 10c268d6da33b0a4590a478c0f32fd70, 11abc6d6caf592b351052ae8adf50fd1, 1a492bcf61cc6ef0604137033fc7c90b, 43fd53cccd0bd76519c89a78d6bf6131, 5ff24ce91fac66115879ac4ec57a346a, 80710108d60e6f4a1b8975ccfe51e50e, 82c0a84200b3abe2cb346258143147c7, 84316f2c5b758522338a907d7e8f386d, 89e8c9d6d5af67c75803894f4aea8ecc, 98cb12bd0a37bbbfee08e3459a85f735, 9bd21adcd80ce6ee8c5ed1a581c04749, a840e8f158b2eaf5c08595bd38db426e, e86ecaa5f83edede0f153db264078a94 2024-11-13T05:58:27,785 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T05:58:27,786 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-13T05:58:27,786 DEBUG [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 19321613b69704e42207868d450f785b, 200f36be3ca8364314512c33e4ae22d6, 2810327548325125e764254e87017046, 408b59e776ffab17a3ee0e8fb465c67c, 408dfc042fd5f2d964a15b01fd44966c, 4b613e7fdd93a34284c913bfa5a7b222, 90b03edd8043eabbad2ddbd32409c478, 9afb90ba9af49bb3547d4438986e1fca, a65241dbe6c2863f31a0138a73130225, b540ac2e8c4575b424d481848c4b4e52, e903b94793c361b461207b7aa3c133ac, f7c7452ce5410ec8dcf456adb9b7db6b, ff62c0b5be01546e2b1563e065750a3b 2024-11-13T05:58:27,786 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:58:27,786 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:58:27,786 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:58:27,786 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:58:27,786 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:58:27,786 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=13.37 KB heapSize=19.67 KB 2024-11-13T05:58:27,797 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/.tmp/info/a8ce5133401d40a895a1781aaf40b57b is 268, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac./info:regioninfo/1731477505330/Put/seqid=0 2024-11-13T05:58:27,807 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/90b03edd8043eabbad2ddbd32409c478/.tmp/colfamily11/4bf0088df5d648ca81cab772b22bb7a9 is 52, key is zzz/colfamily11:colfamily11/1731477320816/Put/seqid=0 2024-11-13T05:58:27,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741887_1063 (size=19812) 2024-11-13T05:58:27,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741887_1063 (size=19812) 2024-11-13T05:58:27,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741887_1063 (size=19812) 2024-11-13T05:58:27,812 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.37 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/.tmp/info/a8ce5133401d40a895a1781aaf40b57b 2024-11-13T05:58:27,814 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a840e8f158b2eaf5c08595bd38db426e/.tmp/colfamily11/83b187a1bf3d4db5b41d6957d550963e is 52, key is ccc/colfamily11:colfamily11/1731477320698/Put/seqid=0 2024-11-13T05:58:27,820 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/.tmp/info/a8ce5133401d40a895a1781aaf40b57b as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/info/a8ce5133401d40a895a1781aaf40b57b 2024-11-13T05:58:27,821 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T05:58:27,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741888_1064 (size=4857) 2024-11-13T05:58:27,826 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/90b03edd8043eabbad2ddbd32409c478/.tmp/colfamily11/4bf0088df5d648ca81cab772b22bb7a9 2024-11-13T05:58:27,828 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/info/a8ce5133401d40a895a1781aaf40b57b, entries=72, sequenceid=81, filesize=19.3 K 2024-11-13T05:58:27,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741888_1064 (size=4857) 2024-11-13T05:58:27,829 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~13.37 KB/13686, heapSize ~18.91 KB/19360, currentSize=0 B/0 for 1588230740 in 43ms, sequenceid=81, compaction requested=false 2024-11-13T05:58:27,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741888_1064 (size=4857) 2024-11-13T05:58:27,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741889_1065 (size=39263) 2024-11-13T05:58:27,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741889_1065 (size=39263) 2024-11-13T05:58:27,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741889_1065 (size=39263) 2024-11-13T05:58:27,837 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/90b03edd8043eabbad2ddbd32409c478/.tmp/colfamily11/4bf0088df5d648ca81cab772b22bb7a9 as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/90b03edd8043eabbad2ddbd32409c478/colfamily11/4bf0088df5d648ca81cab772b22bb7a9 2024-11-13T05:58:27,839 INFO [regionserver/bc386b7bf2fe:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T05:58:27,839 INFO [regionserver/bc386b7bf2fe:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T05:58:27,840 INFO [regionserver/bc386b7bf2fe:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T05:58:27,840 INFO [regionserver/bc386b7bf2fe:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T05:58:27,847 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a840e8f158b2eaf5c08595bd38db426e/.tmp/colfamily11/83b187a1bf3d4db5b41d6957d550963e 2024-11-13T05:58:27,848 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/hbase/meta/1588230740/recovered.edits/84.seqid, newMaxSeqId=84, maxSeqId=60 2024-11-13T05:58:27,851 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:58:27,851 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:58:27,851 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731477507786Running coprocessor pre-close hooks at 1731477507786Disabling compacts and flushes for region at 1731477507786Disabling writes for close at 1731477507786Obtaining lock to block concurrent updates at 1731477507786Preparing flush snapshotting stores in 1588230740 at 1731477507786Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=13686, getHeapSize=20080, getOffHeapSize=0, getCellsCount=72 at 1731477507791 (+5 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731477507792 (+1 ms)Flushing 1588230740/info: creating writer at 1731477507792Flushing 1588230740/info: appending metadata at 1731477507797 (+5 ms)Flushing 1588230740/info: closing flushed file at 1731477507797Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68c0804c: reopening flushed file at 1731477507818 (+21 ms)Finished flush of dataSize ~13.37 KB/13686, heapSize ~18.91 KB/19360, currentSize=0 B/0 for 1588230740 in 43ms, sequenceid=81, compaction requested=false at 1731477507829 (+11 ms)Writing region close event to WAL at 1731477507843 (+14 ms)Running coprocessor post-close hooks at 1731477507849 (+6 ms)Closed at 1731477507851 (+2 ms) 2024-11-13T05:58:27,852 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T05:58:27,854 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/90b03edd8043eabbad2ddbd32409c478/colfamily11/4bf0088df5d648ca81cab772b22bb7a9, entries=1, sequenceid=5, filesize=4.7 K 2024-11-13T05:58:27,855 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~48 B/48, heapSize ~360 B/360, currentSize=0 B/0 for 90b03edd8043eabbad2ddbd32409c478 in 70ms, sequenceid=5, compaction requested=false 2024-11-13T05:58:27,855 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a840e8f158b2eaf5c08595bd38db426e/.tmp/colfamily11/83b187a1bf3d4db5b41d6957d550963e as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a840e8f158b2eaf5c08595bd38db426e/colfamily11/83b187a1bf3d4db5b41d6957d550963e 2024-11-13T05:58:27,859 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/90b03edd8043eabbad2ddbd32409c478/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T05:58:27,859 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478. 2024-11-13T05:58:27,859 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 90b03edd8043eabbad2ddbd32409c478: Waiting for close lock at 1731477507784Running coprocessor pre-close hooks at 1731477507784Disabling compacts and flushes for region at 1731477507784Disabling writes for close at 1731477507784Obtaining lock to block concurrent updates at 1731477507784Preparing flush snapshotting stores in 90b03edd8043eabbad2ddbd32409c478 at 1731477507784Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478., syncing WAL and waiting on mvcc, flushsize=dataSize=48, getHeapSize=840, getOffHeapSize=0, getCellsCount=1 at 1731477507791 (+7 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478. at 1731477507792 (+1 ms)Flushing 90b03edd8043eabbad2ddbd32409c478/colfamily11: creating writer at 1731477507792Flushing 90b03edd8043eabbad2ddbd32409c478/colfamily11: appending metadata at 1731477507806 (+14 ms)Flushing 90b03edd8043eabbad2ddbd32409c478/colfamily11: closing flushed file at 1731477507807 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16fbe1c4: reopening flushed file at 1731477507835 (+28 ms)Finished flush of dataSize ~48 B/48, heapSize ~360 B/360, currentSize=0 B/0 for 90b03edd8043eabbad2ddbd32409c478 in 70ms, sequenceid=5, compaction requested=false at 1731477507855 (+20 ms)Writing region close event to WAL at 1731477507855Running coprocessor post-close hooks at 1731477507859 (+4 ms)Closed at 1731477507859 2024-11-13T05:58:27,860 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731477316849.90b03edd8043eabbad2ddbd32409c478. 2024-11-13T05:58:27,860 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a65241dbe6c2863f31a0138a73130225, disabling compactions & flushes 2024-11-13T05:58:27,860 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225. 2024-11-13T05:58:27,860 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225. 2024-11-13T05:58:27,860 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225. after waiting 0 ms 2024-11-13T05:58:27,860 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225. 2024-11-13T05:58:27,862 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a840e8f158b2eaf5c08595bd38db426e/colfamily11/83b187a1bf3d4db5b41d6957d550963e, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T05:58:27,863 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for a840e8f158b2eaf5c08595bd38db426e in 79ms, sequenceid=5, compaction requested=false 2024-11-13T05:58:27,865 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a65241dbe6c2863f31a0138a73130225/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T05:58:27,866 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225. 2024-11-13T05:58:27,866 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a65241dbe6c2863f31a0138a73130225: Waiting for close lock at 1731477507860Running coprocessor pre-close hooks at 1731477507860Disabling compacts and flushes for region at 1731477507860Disabling writes for close at 1731477507860Writing region close event to WAL at 1731477507861 (+1 ms)Running coprocessor post-close hooks at 1731477507865 (+4 ms)Closed at 1731477507866 (+1 ms) 2024-11-13T05:58:27,866 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731477316849.a65241dbe6c2863f31a0138a73130225. 2024-11-13T05:58:27,866 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 200f36be3ca8364314512c33e4ae22d6, disabling compactions & flushes 2024-11-13T05:58:27,866 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6. 2024-11-13T05:58:27,866 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6. 2024-11-13T05:58:27,866 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6. after waiting 0 ms 2024-11-13T05:58:27,866 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6. 2024-11-13T05:58:27,866 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 200f36be3ca8364314512c33e4ae22d6 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:27,868 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a840e8f158b2eaf5c08595bd38db426e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T05:58:27,869 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e. 2024-11-13T05:58:27,869 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a840e8f158b2eaf5c08595bd38db426e: Waiting for close lock at 1731477507783Running coprocessor pre-close hooks at 1731477507784 (+1 ms)Disabling compacts and flushes for region at 1731477507784Disabling writes for close at 1731477507784Obtaining lock to block concurrent updates at 1731477507784Preparing flush snapshotting stores in a840e8f158b2eaf5c08595bd38db426e at 1731477507784Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477507791 (+7 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e. at 1731477507792 (+1 ms)Flushing a840e8f158b2eaf5c08595bd38db426e/colfamily11: creating writer at 1731477507792Flushing a840e8f158b2eaf5c08595bd38db426e/colfamily11: appending metadata at 1731477507813 (+21 ms)Flushing a840e8f158b2eaf5c08595bd38db426e/colfamily11: closing flushed file at 1731477507814 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3753fbad: reopening flushed file at 1731477507853 (+39 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for a840e8f158b2eaf5c08595bd38db426e in 79ms, sequenceid=5, compaction requested=false at 1731477507863 (+10 ms)Writing region close event to WAL at 1731477507863Running coprocessor post-close hooks at 1731477507869 (+6 ms)Closed at 1731477507869 2024-11-13T05:58:27,869 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731477316849.a840e8f158b2eaf5c08595bd38db426e. 2024-11-13T05:58:27,870 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 43fd53cccd0bd76519c89a78d6bf6131, disabling compactions & flushes 2024-11-13T05:58:27,870 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131. 2024-11-13T05:58:27,870 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131. 2024-11-13T05:58:27,870 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131. after waiting 0 ms 2024-11-13T05:58:27,870 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131. 2024-11-13T05:58:27,870 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 43fd53cccd0bd76519c89a78d6bf6131 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:27,883 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/200f36be3ca8364314512c33e4ae22d6/.tmp/colfamily11/9b99a158de9042769f37fd3f76fd8e04 is 52, key is bbb/colfamily11:colfamily11/1731477320458/Put/seqid=0 2024-11-13T05:58:27,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741890_1066 (size=39263) 2024-11-13T05:58:27,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741890_1066 (size=39263) 2024-11-13T05:58:27,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741890_1066 (size=39263) 2024-11-13T05:58:27,890 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/200f36be3ca8364314512c33e4ae22d6/.tmp/colfamily11/9b99a158de9042769f37fd3f76fd8e04 2024-11-13T05:58:27,893 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43fd53cccd0bd76519c89a78d6bf6131/.tmp/colfamily11/f742d2c834fc4ab0bd5b8f87e6228534 is 52, key is ooo/colfamily11:colfamily11/1731477320876/Put/seqid=0 2024-11-13T05:58:27,898 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/200f36be3ca8364314512c33e4ae22d6/.tmp/colfamily11/9b99a158de9042769f37fd3f76fd8e04 as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/200f36be3ca8364314512c33e4ae22d6/colfamily11/9b99a158de9042769f37fd3f76fd8e04 2024-11-13T05:58:27,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741891_1067 (size=39263) 2024-11-13T05:58:27,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741891_1067 (size=39263) 2024-11-13T05:58:27,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741891_1067 (size=39263) 2024-11-13T05:58:27,903 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43fd53cccd0bd76519c89a78d6bf6131/.tmp/colfamily11/f742d2c834fc4ab0bd5b8f87e6228534 2024-11-13T05:58:27,912 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/200f36be3ca8364314512c33e4ae22d6/colfamily11/9b99a158de9042769f37fd3f76fd8e04, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T05:58:27,914 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 200f36be3ca8364314512c33e4ae22d6 in 47ms, sequenceid=5, compaction requested=false 2024-11-13T05:58:27,917 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43fd53cccd0bd76519c89a78d6bf6131/.tmp/colfamily11/f742d2c834fc4ab0bd5b8f87e6228534 as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43fd53cccd0bd76519c89a78d6bf6131/colfamily11/f742d2c834fc4ab0bd5b8f87e6228534 2024-11-13T05:58:27,919 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/200f36be3ca8364314512c33e4ae22d6/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T05:58:27,920 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6. 2024-11-13T05:58:27,920 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 200f36be3ca8364314512c33e4ae22d6: Waiting for close lock at 1731477507866Running coprocessor pre-close hooks at 1731477507866Disabling compacts and flushes for region at 1731477507866Disabling writes for close at 1731477507866Obtaining lock to block concurrent updates at 1731477507866Preparing flush snapshotting stores in 200f36be3ca8364314512c33e4ae22d6 at 1731477507866Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477507867 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6. at 1731477507867Flushing 200f36be3ca8364314512c33e4ae22d6/colfamily11: creating writer at 1731477507867Flushing 200f36be3ca8364314512c33e4ae22d6/colfamily11: appending metadata at 1731477507883 (+16 ms)Flushing 200f36be3ca8364314512c33e4ae22d6/colfamily11: closing flushed file at 1731477507883Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e8a6eed: reopening flushed file at 1731477507897 (+14 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 200f36be3ca8364314512c33e4ae22d6 in 47ms, sequenceid=5, compaction requested=false at 1731477507914 (+17 ms)Writing region close event to WAL at 1731477507915 (+1 ms)Running coprocessor post-close hooks at 1731477507920 (+5 ms)Closed at 1731477507920 2024-11-13T05:58:27,920 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731477316849.200f36be3ca8364314512c33e4ae22d6. 2024-11-13T05:58:27,920 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ff62c0b5be01546e2b1563e065750a3b, disabling compactions & flushes 2024-11-13T05:58:27,920 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b. 2024-11-13T05:58:27,920 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b. 2024-11-13T05:58:27,920 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b. after waiting 0 ms 2024-11-13T05:58:27,920 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b. 2024-11-13T05:58:27,920 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ff62c0b5be01546e2b1563e065750a3b 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:27,926 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43fd53cccd0bd76519c89a78d6bf6131/colfamily11/f742d2c834fc4ab0bd5b8f87e6228534, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T05:58:27,927 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 43fd53cccd0bd76519c89a78d6bf6131 in 57ms, sequenceid=5, compaction requested=false 2024-11-13T05:58:27,932 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/43fd53cccd0bd76519c89a78d6bf6131/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T05:58:27,933 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131. 2024-11-13T05:58:27,933 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 43fd53cccd0bd76519c89a78d6bf6131: Waiting for close lock at 1731477507869Running coprocessor pre-close hooks at 1731477507869Disabling compacts and flushes for region at 1731477507869Disabling writes for close at 1731477507870 (+1 ms)Obtaining lock to block concurrent updates at 1731477507870Preparing flush snapshotting stores in 43fd53cccd0bd76519c89a78d6bf6131 at 1731477507870Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477507870Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131. at 1731477507871 (+1 ms)Flushing 43fd53cccd0bd76519c89a78d6bf6131/colfamily11: creating writer at 1731477507871Flushing 43fd53cccd0bd76519c89a78d6bf6131/colfamily11: appending metadata at 1731477507892 (+21 ms)Flushing 43fd53cccd0bd76519c89a78d6bf6131/colfamily11: closing flushed file at 1731477507892Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f356555: reopening flushed file at 1731477507917 (+25 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 43fd53cccd0bd76519c89a78d6bf6131 in 57ms, sequenceid=5, compaction requested=false at 1731477507927 (+10 ms)Writing region close event to WAL at 1731477507928 (+1 ms)Running coprocessor post-close hooks at 1731477507933 (+5 ms)Closed at 1731477507933 2024-11-13T05:58:27,933 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731477316849.43fd53cccd0bd76519c89a78d6bf6131. 2024-11-13T05:58:27,933 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 89e8c9d6d5af67c75803894f4aea8ecc, disabling compactions & flushes 2024-11-13T05:58:27,933 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc. 2024-11-13T05:58:27,933 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc. 2024-11-13T05:58:27,933 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc. after waiting 0 ms 2024-11-13T05:58:27,933 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc. 2024-11-13T05:58:27,933 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 89e8c9d6d5af67c75803894f4aea8ecc 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:27,939 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ff62c0b5be01546e2b1563e065750a3b/.tmp/colfamily11/270637c30c404fe6999e3ff27d29974a is 52, key is www/colfamily11:colfamily11/1731477320802/Put/seqid=0 2024-11-13T05:58:27,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741892_1068 (size=39263) 2024-11-13T05:58:27,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741892_1068 (size=39263) 2024-11-13T05:58:27,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741892_1068 (size=39263) 2024-11-13T05:58:27,951 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ff62c0b5be01546e2b1563e065750a3b/.tmp/colfamily11/270637c30c404fe6999e3ff27d29974a 2024-11-13T05:58:27,957 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/89e8c9d6d5af67c75803894f4aea8ecc/.tmp/colfamily11/e4ad9dd02c1e41f28a4f2b85c867843e is 52, key is iii/colfamily11:colfamily11/1731477320811/Put/seqid=0 2024-11-13T05:58:27,958 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ff62c0b5be01546e2b1563e065750a3b/.tmp/colfamily11/270637c30c404fe6999e3ff27d29974a as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ff62c0b5be01546e2b1563e065750a3b/colfamily11/270637c30c404fe6999e3ff27d29974a 2024-11-13T05:58:27,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741893_1069 (size=39263) 2024-11-13T05:58:27,966 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ff62c0b5be01546e2b1563e065750a3b/colfamily11/270637c30c404fe6999e3ff27d29974a, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T05:58:27,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741893_1069 (size=39263) 2024-11-13T05:58:27,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741893_1069 (size=39263) 2024-11-13T05:58:27,968 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/89e8c9d6d5af67c75803894f4aea8ecc/.tmp/colfamily11/e4ad9dd02c1e41f28a4f2b85c867843e 2024-11-13T05:58:27,969 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for ff62c0b5be01546e2b1563e065750a3b in 49ms, sequenceid=5, compaction requested=false 2024-11-13T05:58:27,974 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ff62c0b5be01546e2b1563e065750a3b/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T05:58:27,975 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b. 2024-11-13T05:58:27,975 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/89e8c9d6d5af67c75803894f4aea8ecc/.tmp/colfamily11/e4ad9dd02c1e41f28a4f2b85c867843e as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/89e8c9d6d5af67c75803894f4aea8ecc/colfamily11/e4ad9dd02c1e41f28a4f2b85c867843e 2024-11-13T05:58:27,975 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ff62c0b5be01546e2b1563e065750a3b: Waiting for close lock at 1731477507920Running coprocessor pre-close hooks at 1731477507920Disabling compacts and flushes for region at 1731477507920Disabling writes for close at 1731477507920Obtaining lock to block concurrent updates at 1731477507920Preparing flush snapshotting stores in ff62c0b5be01546e2b1563e065750a3b at 1731477507920Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477507921 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b. at 1731477507921Flushing ff62c0b5be01546e2b1563e065750a3b/colfamily11: creating writer at 1731477507921Flushing ff62c0b5be01546e2b1563e065750a3b/colfamily11: appending metadata at 1731477507939 (+18 ms)Flushing ff62c0b5be01546e2b1563e065750a3b/colfamily11: closing flushed file at 1731477507939Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59faf79d: reopening flushed file at 1731477507957 (+18 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for ff62c0b5be01546e2b1563e065750a3b in 49ms, sequenceid=5, compaction requested=false at 1731477507969 (+12 ms)Writing region close event to WAL at 1731477507970 (+1 ms)Running coprocessor post-close hooks at 1731477507975 (+5 ms)Closed at 1731477507975 2024-11-13T05:58:27,975 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731477316849.ff62c0b5be01546e2b1563e065750a3b. 2024-11-13T05:58:27,976 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4b613e7fdd93a34284c913bfa5a7b222, disabling compactions & flushes 2024-11-13T05:58:27,976 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222. 2024-11-13T05:58:27,976 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222. 2024-11-13T05:58:27,976 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222. after waiting 0 ms 2024-11-13T05:58:27,976 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222. 2024-11-13T05:58:27,980 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4b613e7fdd93a34284c913bfa5a7b222/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T05:58:27,981 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222. 2024-11-13T05:58:27,981 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4b613e7fdd93a34284c913bfa5a7b222: Waiting for close lock at 1731477507976Running coprocessor pre-close hooks at 1731477507976Disabling compacts and flushes for region at 1731477507976Disabling writes for close at 1731477507976Writing region close event to WAL at 1731477507977 (+1 ms)Running coprocessor post-close hooks at 1731477507981 (+4 ms)Closed at 1731477507981 2024-11-13T05:58:27,981 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731477316849.4b613e7fdd93a34284c913bfa5a7b222. 2024-11-13T05:58:27,981 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 19321613b69704e42207868d450f785b, disabling compactions & flushes 2024-11-13T05:58:27,981 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b. 2024-11-13T05:58:27,981 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b. 2024-11-13T05:58:27,981 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b. after waiting 0 ms 2024-11-13T05:58:27,981 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b. 2024-11-13T05:58:27,981 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 19321613b69704e42207868d450f785b 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:27,983 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/89e8c9d6d5af67c75803894f4aea8ecc/colfamily11/e4ad9dd02c1e41f28a4f2b85c867843e, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T05:58:27,984 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 89e8c9d6d5af67c75803894f4aea8ecc in 51ms, sequenceid=5, compaction requested=false 2024-11-13T05:58:27,986 DEBUG [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(1351): Waiting on 10c268d6da33b0a4590a478c0f32fd70, 11abc6d6caf592b351052ae8adf50fd1, 1a492bcf61cc6ef0604137033fc7c90b, 5ff24ce91fac66115879ac4ec57a346a, 80710108d60e6f4a1b8975ccfe51e50e, 82c0a84200b3abe2cb346258143147c7, 84316f2c5b758522338a907d7e8f386d, 89e8c9d6d5af67c75803894f4aea8ecc, 98cb12bd0a37bbbfee08e3459a85f735, 9bd21adcd80ce6ee8c5ed1a581c04749, e86ecaa5f83edede0f153db264078a94 2024-11-13T05:58:27,986 DEBUG [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(1351): Waiting on 19321613b69704e42207868d450f785b, 2810327548325125e764254e87017046, 408b59e776ffab17a3ee0e8fb465c67c, 408dfc042fd5f2d964a15b01fd44966c, 9afb90ba9af49bb3547d4438986e1fca, b540ac2e8c4575b424d481848c4b4e52, e903b94793c361b461207b7aa3c133ac, f7c7452ce5410ec8dcf456adb9b7db6b 2024-11-13T05:58:27,988 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/89e8c9d6d5af67c75803894f4aea8ecc/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T05:58:27,989 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc. 2024-11-13T05:58:27,989 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 89e8c9d6d5af67c75803894f4aea8ecc: Waiting for close lock at 1731477507933Running coprocessor pre-close hooks at 1731477507933Disabling compacts and flushes for region at 1731477507933Disabling writes for close at 1731477507933Obtaining lock to block concurrent updates at 1731477507933Preparing flush snapshotting stores in 89e8c9d6d5af67c75803894f4aea8ecc at 1731477507933Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477507934 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc. at 1731477507934Flushing 89e8c9d6d5af67c75803894f4aea8ecc/colfamily11: creating writer at 1731477507934Flushing 89e8c9d6d5af67c75803894f4aea8ecc/colfamily11: appending metadata at 1731477507956 (+22 ms)Flushing 89e8c9d6d5af67c75803894f4aea8ecc/colfamily11: closing flushed file at 1731477507956Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b250e39: reopening flushed file at 1731477507975 (+19 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 89e8c9d6d5af67c75803894f4aea8ecc in 51ms, sequenceid=5, compaction requested=false at 1731477507984 (+9 ms)Writing region close event to WAL at 1731477507985 (+1 ms)Running coprocessor post-close hooks at 1731477507989 (+4 ms)Closed at 1731477507989 2024-11-13T05:58:27,989 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731477316849.89e8c9d6d5af67c75803894f4aea8ecc. 2024-11-13T05:58:27,989 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9bd21adcd80ce6ee8c5ed1a581c04749, disabling compactions & flushes 2024-11-13T05:58:27,989 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749. 2024-11-13T05:58:27,989 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749. 2024-11-13T05:58:27,989 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749. after waiting 0 ms 2024-11-13T05:58:27,989 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749. 2024-11-13T05:58:27,989 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9bd21adcd80ce6ee8c5ed1a581c04749 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:28,003 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/19321613b69704e42207868d450f785b/.tmp/colfamily11/c6473cc9f21a4bf3898aead4f12b11d4 is 52, key is eee/colfamily11:colfamily11/1731477320506/Put/seqid=0 2024-11-13T05:58:28,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741894_1070 (size=39263) 2024-11-13T05:58:28,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741894_1070 (size=39263) 2024-11-13T05:58:28,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741894_1070 (size=39263) 2024-11-13T05:58:28,011 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9bd21adcd80ce6ee8c5ed1a581c04749/.tmp/colfamily11/09074063dcbd4f4eb3bf19459d2b1d6b is 52, key is rrr/colfamily11:colfamily11/1731477320910/Put/seqid=0 2024-11-13T05:58:28,012 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/19321613b69704e42207868d450f785b/.tmp/colfamily11/c6473cc9f21a4bf3898aead4f12b11d4 2024-11-13T05:58:28,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741895_1071 (size=39263) 2024-11-13T05:58:28,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741895_1071 (size=39263) 2024-11-13T05:58:28,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741895_1071 (size=39263) 2024-11-13T05:58:28,021 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/19321613b69704e42207868d450f785b/.tmp/colfamily11/c6473cc9f21a4bf3898aead4f12b11d4 as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/19321613b69704e42207868d450f785b/colfamily11/c6473cc9f21a4bf3898aead4f12b11d4 2024-11-13T05:58:28,021 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9bd21adcd80ce6ee8c5ed1a581c04749/.tmp/colfamily11/09074063dcbd4f4eb3bf19459d2b1d6b 2024-11-13T05:58:28,028 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/19321613b69704e42207868d450f785b/colfamily11/c6473cc9f21a4bf3898aead4f12b11d4, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T05:58:28,029 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9bd21adcd80ce6ee8c5ed1a581c04749/.tmp/colfamily11/09074063dcbd4f4eb3bf19459d2b1d6b as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9bd21adcd80ce6ee8c5ed1a581c04749/colfamily11/09074063dcbd4f4eb3bf19459d2b1d6b 2024-11-13T05:58:28,029 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 19321613b69704e42207868d450f785b in 48ms, sequenceid=5, compaction requested=false 2024-11-13T05:58:28,034 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/19321613b69704e42207868d450f785b/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T05:58:28,035 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b. 2024-11-13T05:58:28,035 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 19321613b69704e42207868d450f785b: Waiting for close lock at 1731477507981Running coprocessor pre-close hooks at 1731477507981Disabling compacts and flushes for region at 1731477507981Disabling writes for close at 1731477507981Obtaining lock to block concurrent updates at 1731477507981Preparing flush snapshotting stores in 19321613b69704e42207868d450f785b at 1731477507981Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477507982 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b. at 1731477507982Flushing 19321613b69704e42207868d450f785b/colfamily11: creating writer at 1731477507982Flushing 19321613b69704e42207868d450f785b/colfamily11: appending metadata at 1731477508003 (+21 ms)Flushing 19321613b69704e42207868d450f785b/colfamily11: closing flushed file at 1731477508003Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33200742: reopening flushed file at 1731477508020 (+17 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 19321613b69704e42207868d450f785b in 48ms, sequenceid=5, compaction requested=false at 1731477508029 (+9 ms)Writing region close event to WAL at 1731477508030 (+1 ms)Running coprocessor post-close hooks at 1731477508035 (+5 ms)Closed at 1731477508035 2024-11-13T05:58:28,036 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731477316849.19321613b69704e42207868d450f785b. 2024-11-13T05:58:28,036 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9afb90ba9af49bb3547d4438986e1fca, disabling compactions & flushes 2024-11-13T05:58:28,036 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca. 2024-11-13T05:58:28,036 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca. 2024-11-13T05:58:28,036 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca. after waiting 0 ms 2024-11-13T05:58:28,036 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca. 2024-11-13T05:58:28,036 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9bd21adcd80ce6ee8c5ed1a581c04749/colfamily11/09074063dcbd4f4eb3bf19459d2b1d6b, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T05:58:28,037 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 9bd21adcd80ce6ee8c5ed1a581c04749 in 48ms, sequenceid=5, compaction requested=false 2024-11-13T05:58:28,040 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9afb90ba9af49bb3547d4438986e1fca/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T05:58:28,041 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca. 2024-11-13T05:58:28,041 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9afb90ba9af49bb3547d4438986e1fca: Waiting for close lock at 1731477508036Running coprocessor pre-close hooks at 1731477508036Disabling compacts and flushes for region at 1731477508036Disabling writes for close at 1731477508036Writing region close event to WAL at 1731477508037 (+1 ms)Running coprocessor post-close hooks at 1731477508041 (+4 ms)Closed at 1731477508041 2024-11-13T05:58:28,041 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731477316849.9afb90ba9af49bb3547d4438986e1fca. 2024-11-13T05:58:28,041 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f7c7452ce5410ec8dcf456adb9b7db6b, disabling compactions & flushes 2024-11-13T05:58:28,042 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b. 2024-11-13T05:58:28,042 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9bd21adcd80ce6ee8c5ed1a581c04749/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T05:58:28,042 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b. 2024-11-13T05:58:28,042 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b. after waiting 0 ms 2024-11-13T05:58:28,042 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b. 2024-11-13T05:58:28,042 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f7c7452ce5410ec8dcf456adb9b7db6b 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:28,042 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749. 2024-11-13T05:58:28,042 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9bd21adcd80ce6ee8c5ed1a581c04749: Waiting for close lock at 1731477507989Running coprocessor pre-close hooks at 1731477507989Disabling compacts and flushes for region at 1731477507989Disabling writes for close at 1731477507989Obtaining lock to block concurrent updates at 1731477507989Preparing flush snapshotting stores in 9bd21adcd80ce6ee8c5ed1a581c04749 at 1731477507989Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477507990 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749. at 1731477507990Flushing 9bd21adcd80ce6ee8c5ed1a581c04749/colfamily11: creating writer at 1731477507990Flushing 9bd21adcd80ce6ee8c5ed1a581c04749/colfamily11: appending metadata at 1731477508011 (+21 ms)Flushing 9bd21adcd80ce6ee8c5ed1a581c04749/colfamily11: closing flushed file at 1731477508011Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58d75e2b: reopening flushed file at 1731477508028 (+17 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 9bd21adcd80ce6ee8c5ed1a581c04749 in 48ms, sequenceid=5, compaction requested=false at 1731477508038 (+10 ms)Writing region close event to WAL at 1731477508038Running coprocessor post-close hooks at 1731477508042 (+4 ms)Closed at 1731477508042 2024-11-13T05:58:28,042 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731477316849.9bd21adcd80ce6ee8c5ed1a581c04749. 2024-11-13T05:58:28,042 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1a492bcf61cc6ef0604137033fc7c90b, disabling compactions & flushes 2024-11-13T05:58:28,043 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b. 2024-11-13T05:58:28,043 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b. 2024-11-13T05:58:28,043 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b. after waiting 0 ms 2024-11-13T05:58:28,043 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b. 2024-11-13T05:58:28,043 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1a492bcf61cc6ef0604137033fc7c90b 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:28,063 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1a492bcf61cc6ef0604137033fc7c90b/.tmp/colfamily11/31ba1313185b43d1af0d65614e017cb1 is 52, key is fff/colfamily11:colfamily11/1731477320774/Put/seqid=0 2024-11-13T05:58:28,064 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f7c7452ce5410ec8dcf456adb9b7db6b/.tmp/colfamily11/d61e8b11fa294f668728c43e3978a409 is 52, key is nnn/colfamily11:colfamily11/1731477320689/Put/seqid=0 2024-11-13T05:58:28,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741897_1073 (size=39263) 2024-11-13T05:58:28,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741896_1072 (size=39263) 2024-11-13T05:58:28,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741897_1073 (size=39263) 2024-11-13T05:58:28,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741897_1073 (size=39263) 2024-11-13T05:58:28,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741896_1072 (size=39263) 2024-11-13T05:58:28,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741896_1072 (size=39263) 2024-11-13T05:58:28,075 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f7c7452ce5410ec8dcf456adb9b7db6b/.tmp/colfamily11/d61e8b11fa294f668728c43e3978a409 2024-11-13T05:58:28,075 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1a492bcf61cc6ef0604137033fc7c90b/.tmp/colfamily11/31ba1313185b43d1af0d65614e017cb1 2024-11-13T05:58:28,082 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1a492bcf61cc6ef0604137033fc7c90b/.tmp/colfamily11/31ba1313185b43d1af0d65614e017cb1 as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1a492bcf61cc6ef0604137033fc7c90b/colfamily11/31ba1313185b43d1af0d65614e017cb1 2024-11-13T05:58:28,082 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f7c7452ce5410ec8dcf456adb9b7db6b/.tmp/colfamily11/d61e8b11fa294f668728c43e3978a409 as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f7c7452ce5410ec8dcf456adb9b7db6b/colfamily11/d61e8b11fa294f668728c43e3978a409 2024-11-13T05:58:28,088 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1a492bcf61cc6ef0604137033fc7c90b/colfamily11/31ba1313185b43d1af0d65614e017cb1, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T05:58:28,088 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f7c7452ce5410ec8dcf456adb9b7db6b/colfamily11/d61e8b11fa294f668728c43e3978a409, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T05:58:28,089 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for f7c7452ce5410ec8dcf456adb9b7db6b in 47ms, sequenceid=5, compaction requested=false 2024-11-13T05:58:28,089 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 1a492bcf61cc6ef0604137033fc7c90b in 46ms, sequenceid=5, compaction requested=false 2024-11-13T05:58:28,093 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1a492bcf61cc6ef0604137033fc7c90b/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T05:58:28,093 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f7c7452ce5410ec8dcf456adb9b7db6b/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T05:58:28,094 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b. 2024-11-13T05:58:28,094 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1a492bcf61cc6ef0604137033fc7c90b: Waiting for close lock at 1731477508042Running coprocessor pre-close hooks at 1731477508042Disabling compacts and flushes for region at 1731477508042Disabling writes for close at 1731477508043 (+1 ms)Obtaining lock to block concurrent updates at 1731477508043Preparing flush snapshotting stores in 1a492bcf61cc6ef0604137033fc7c90b at 1731477508043Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477508043Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b. at 1731477508044 (+1 ms)Flushing 1a492bcf61cc6ef0604137033fc7c90b/colfamily11: creating writer at 1731477508044Flushing 1a492bcf61cc6ef0604137033fc7c90b/colfamily11: appending metadata at 1731477508063 (+19 ms)Flushing 1a492bcf61cc6ef0604137033fc7c90b/colfamily11: closing flushed file at 1731477508063Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ad27d9b: reopening flushed file at 1731477508081 (+18 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 1a492bcf61cc6ef0604137033fc7c90b in 46ms, sequenceid=5, compaction requested=false at 1731477508089 (+8 ms)Writing region close event to WAL at 1731477508090 (+1 ms)Running coprocessor post-close hooks at 1731477508094 (+4 ms)Closed at 1731477508094 2024-11-13T05:58:28,094 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b. 2024-11-13T05:58:28,094 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f7c7452ce5410ec8dcf456adb9b7db6b: Waiting for close lock at 1731477508041Running coprocessor pre-close hooks at 1731477508041Disabling compacts and flushes for region at 1731477508041Disabling writes for close at 1731477508042 (+1 ms)Obtaining lock to block concurrent updates at 1731477508042Preparing flush snapshotting stores in f7c7452ce5410ec8dcf456adb9b7db6b at 1731477508042Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477508042Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b. at 1731477508043 (+1 ms)Flushing f7c7452ce5410ec8dcf456adb9b7db6b/colfamily11: creating writer at 1731477508043Flushing f7c7452ce5410ec8dcf456adb9b7db6b/colfamily11: appending metadata at 1731477508063 (+20 ms)Flushing f7c7452ce5410ec8dcf456adb9b7db6b/colfamily11: closing flushed file at 1731477508063Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14c1129b: reopening flushed file at 1731477508081 (+18 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for f7c7452ce5410ec8dcf456adb9b7db6b in 47ms, sequenceid=5, compaction requested=false at 1731477508089 (+8 ms)Writing region close event to WAL at 1731477508090 (+1 ms)Running coprocessor post-close hooks at 1731477508094 (+4 ms)Closed at 1731477508094 2024-11-13T05:58:28,094 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731477316849.1a492bcf61cc6ef0604137033fc7c90b. 2024-11-13T05:58:28,094 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731477316849.f7c7452ce5410ec8dcf456adb9b7db6b. 2024-11-13T05:58:28,094 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5ff24ce91fac66115879ac4ec57a346a, disabling compactions & flushes 2024-11-13T05:58:28,094 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e903b94793c361b461207b7aa3c133ac, disabling compactions & flushes 2024-11-13T05:58:28,094 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a. 2024-11-13T05:58:28,095 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac. 2024-11-13T05:58:28,095 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a. 2024-11-13T05:58:28,095 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac. 2024-11-13T05:58:28,095 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a. after waiting 0 ms 2024-11-13T05:58:28,095 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac. after waiting 0 ms 2024-11-13T05:58:28,095 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a. 2024-11-13T05:58:28,095 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac. 2024-11-13T05:58:28,100 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e903b94793c361b461207b7aa3c133ac/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T05:58:28,100 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ff24ce91fac66115879ac4ec57a346a/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T05:58:28,101 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a. 2024-11-13T05:58:28,101 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac. 2024-11-13T05:58:28,101 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5ff24ce91fac66115879ac4ec57a346a: Waiting for close lock at 1731477508094Running coprocessor pre-close hooks at 1731477508094Disabling compacts and flushes for region at 1731477508094Disabling writes for close at 1731477508095 (+1 ms)Writing region close event to WAL at 1731477508095Running coprocessor post-close hooks at 1731477508101 (+6 ms)Closed at 1731477508101 2024-11-13T05:58:28,101 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e903b94793c361b461207b7aa3c133ac: Waiting for close lock at 1731477508094Running coprocessor pre-close hooks at 1731477508094Disabling compacts and flushes for region at 1731477508094Disabling writes for close at 1731477508095 (+1 ms)Writing region close event to WAL at 1731477508096 (+1 ms)Running coprocessor post-close hooks at 1731477508101 (+5 ms)Closed at 1731477508101 2024-11-13T05:58:28,101 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731477316849.e903b94793c361b461207b7aa3c133ac. 2024-11-13T05:58:28,101 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731477316849.5ff24ce91fac66115879ac4ec57a346a. 2024-11-13T05:58:28,101 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2810327548325125e764254e87017046, disabling compactions & flushes 2024-11-13T05:58:28,101 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 84316f2c5b758522338a907d7e8f386d, disabling compactions & flushes 2024-11-13T05:58:28,101 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046. 2024-11-13T05:58:28,101 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d. 2024-11-13T05:58:28,101 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046. 2024-11-13T05:58:28,101 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d. 2024-11-13T05:58:28,101 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046. after waiting 0 ms 2024-11-13T05:58:28,101 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d. after waiting 0 ms 2024-11-13T05:58:28,101 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046. 2024-11-13T05:58:28,101 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d. 2024-11-13T05:58:28,102 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 2810327548325125e764254e87017046 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:28,106 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/84316f2c5b758522338a907d7e8f386d/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T05:58:28,107 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d. 2024-11-13T05:58:28,107 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 84316f2c5b758522338a907d7e8f386d: Waiting for close lock at 1731477508101Running coprocessor pre-close hooks at 1731477508101Disabling compacts and flushes for region at 1731477508101Disabling writes for close at 1731477508101Writing region close event to WAL at 1731477508102 (+1 ms)Running coprocessor post-close hooks at 1731477508107 (+5 ms)Closed at 1731477508107 2024-11-13T05:58:28,107 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731477316849.84316f2c5b758522338a907d7e8f386d. 2024-11-13T05:58:28,107 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e86ecaa5f83edede0f153db264078a94, disabling compactions & flushes 2024-11-13T05:58:28,107 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94. 2024-11-13T05:58:28,107 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94. 2024-11-13T05:58:28,107 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94. after waiting 0 ms 2024-11-13T05:58:28,108 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94. 2024-11-13T05:58:28,108 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e86ecaa5f83edede0f153db264078a94 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:28,119 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2810327548325125e764254e87017046/.tmp/colfamily11/0da4380c038040ebaee2f6b9562022f9 is 52, key is ttt/colfamily11:colfamily11/1731477320764/Put/seqid=0 2024-11-13T05:58:28,124 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e86ecaa5f83edede0f153db264078a94/.tmp/colfamily11/bd219c9e7b9641f6b932db4a13f2a97a is 52, key is uuu/colfamily11:colfamily11/1731477320935/Put/seqid=0 2024-11-13T05:58:28,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741898_1074 (size=39263) 2024-11-13T05:58:28,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741898_1074 (size=39263) 2024-11-13T05:58:28,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741898_1074 (size=39263) 2024-11-13T05:58:28,134 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2810327548325125e764254e87017046/.tmp/colfamily11/0da4380c038040ebaee2f6b9562022f9 2024-11-13T05:58:28,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741899_1075 (size=39263) 2024-11-13T05:58:28,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741899_1075 (size=39263) 2024-11-13T05:58:28,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741899_1075 (size=39263) 2024-11-13T05:58:28,139 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e86ecaa5f83edede0f153db264078a94/.tmp/colfamily11/bd219c9e7b9641f6b932db4a13f2a97a 2024-11-13T05:58:28,142 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2810327548325125e764254e87017046/.tmp/colfamily11/0da4380c038040ebaee2f6b9562022f9 as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2810327548325125e764254e87017046/colfamily11/0da4380c038040ebaee2f6b9562022f9 2024-11-13T05:58:28,147 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e86ecaa5f83edede0f153db264078a94/.tmp/colfamily11/bd219c9e7b9641f6b932db4a13f2a97a as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e86ecaa5f83edede0f153db264078a94/colfamily11/bd219c9e7b9641f6b932db4a13f2a97a 2024-11-13T05:58:28,149 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2810327548325125e764254e87017046/colfamily11/0da4380c038040ebaee2f6b9562022f9, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T05:58:28,151 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 2810327548325125e764254e87017046 in 49ms, sequenceid=5, compaction requested=false 2024-11-13T05:58:28,155 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e86ecaa5f83edede0f153db264078a94/colfamily11/bd219c9e7b9641f6b932db4a13f2a97a, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T05:58:28,156 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for e86ecaa5f83edede0f153db264078a94 in 48ms, sequenceid=5, compaction requested=false 2024-11-13T05:58:28,156 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2810327548325125e764254e87017046/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T05:58:28,157 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046. 2024-11-13T05:58:28,157 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2810327548325125e764254e87017046: Waiting for close lock at 1731477508101Running coprocessor pre-close hooks at 1731477508101Disabling compacts and flushes for region at 1731477508101Disabling writes for close at 1731477508101Obtaining lock to block concurrent updates at 1731477508102 (+1 ms)Preparing flush snapshotting stores in 2810327548325125e764254e87017046 at 1731477508102Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477508102Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046. at 1731477508103 (+1 ms)Flushing 2810327548325125e764254e87017046/colfamily11: creating writer at 1731477508103Flushing 2810327548325125e764254e87017046/colfamily11: appending metadata at 1731477508119 (+16 ms)Flushing 2810327548325125e764254e87017046/colfamily11: closing flushed file at 1731477508119Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43a4e72f: reopening flushed file at 1731477508141 (+22 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 2810327548325125e764254e87017046 in 49ms, sequenceid=5, compaction requested=false at 1731477508151 (+10 ms)Writing region close event to WAL at 1731477508151Running coprocessor post-close hooks at 1731477508157 (+6 ms)Closed at 1731477508157 2024-11-13T05:58:28,157 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731477316849.2810327548325125e764254e87017046. 2024-11-13T05:58:28,157 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 408dfc042fd5f2d964a15b01fd44966c, disabling compactions & flushes 2024-11-13T05:58:28,157 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c. 2024-11-13T05:58:28,158 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c. 2024-11-13T05:58:28,158 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c. after waiting 0 ms 2024-11-13T05:58:28,158 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c. 2024-11-13T05:58:28,158 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 408dfc042fd5f2d964a15b01fd44966c 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:28,160 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e86ecaa5f83edede0f153db264078a94/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T05:58:28,161 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94. 2024-11-13T05:58:28,161 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e86ecaa5f83edede0f153db264078a94: Waiting for close lock at 1731477508107Running coprocessor pre-close hooks at 1731477508107Disabling compacts and flushes for region at 1731477508107Disabling writes for close at 1731477508107Obtaining lock to block concurrent updates at 1731477508108 (+1 ms)Preparing flush snapshotting stores in e86ecaa5f83edede0f153db264078a94 at 1731477508108Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477508108Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94. at 1731477508109 (+1 ms)Flushing e86ecaa5f83edede0f153db264078a94/colfamily11: creating writer at 1731477508109Flushing e86ecaa5f83edede0f153db264078a94/colfamily11: appending metadata at 1731477508124 (+15 ms)Flushing e86ecaa5f83edede0f153db264078a94/colfamily11: closing flushed file at 1731477508124Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58adad93: reopening flushed file at 1731477508146 (+22 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for e86ecaa5f83edede0f153db264078a94 in 48ms, sequenceid=5, compaction requested=false at 1731477508156 (+10 ms)Writing region close event to WAL at 1731477508157 (+1 ms)Running coprocessor post-close hooks at 1731477508161 (+4 ms)Closed at 1731477508161 2024-11-13T05:58:28,161 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731477316849.e86ecaa5f83edede0f153db264078a94. 2024-11-13T05:58:28,161 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 98cb12bd0a37bbbfee08e3459a85f735, disabling compactions & flushes 2024-11-13T05:58:28,161 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735. 2024-11-13T05:58:28,161 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735. 2024-11-13T05:58:28,161 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735. after waiting 0 ms 2024-11-13T05:58:28,161 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735. 2024-11-13T05:58:28,166 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/98cb12bd0a37bbbfee08e3459a85f735/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T05:58:28,167 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735. 2024-11-13T05:58:28,167 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 98cb12bd0a37bbbfee08e3459a85f735: Waiting for close lock at 1731477508161Running coprocessor pre-close hooks at 1731477508161Disabling compacts and flushes for region at 1731477508161Disabling writes for close at 1731477508161Writing region close event to WAL at 1731477508162 (+1 ms)Running coprocessor post-close hooks at 1731477508166 (+4 ms)Closed at 1731477508167 (+1 ms) 2024-11-13T05:58:28,167 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731477316849.98cb12bd0a37bbbfee08e3459a85f735. 2024-11-13T05:58:28,167 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 80710108d60e6f4a1b8975ccfe51e50e, disabling compactions & flushes 2024-11-13T05:58:28,167 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e. 2024-11-13T05:58:28,167 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e. 2024-11-13T05:58:28,167 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e. after waiting 0 ms 2024-11-13T05:58:28,167 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e. 2024-11-13T05:58:28,167 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 80710108d60e6f4a1b8975ccfe51e50e 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:28,175 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408dfc042fd5f2d964a15b01fd44966c/.tmp/colfamily11/8dd36bd4680e463b995a5658f615b9f8 is 52, key is qqq/colfamily11:colfamily11/1731477320726/Put/seqid=0 2024-11-13T05:58:28,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741900_1076 (size=39263) 2024-11-13T05:58:28,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741900_1076 (size=39263) 2024-11-13T05:58:28,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741900_1076 (size=39263) 2024-11-13T05:58:28,186 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408dfc042fd5f2d964a15b01fd44966c/.tmp/colfamily11/8dd36bd4680e463b995a5658f615b9f8 2024-11-13T05:58:28,186 DEBUG [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(1351): Waiting on 10c268d6da33b0a4590a478c0f32fd70, 11abc6d6caf592b351052ae8adf50fd1, 80710108d60e6f4a1b8975ccfe51e50e, 82c0a84200b3abe2cb346258143147c7 2024-11-13T05:58:28,186 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/80710108d60e6f4a1b8975ccfe51e50e/.tmp/colfamily11/f9d54ed6f80140e5b6da3aa118060251 is 52, key is xxx/colfamily11:colfamily11/1731477320962/Put/seqid=0 2024-11-13T05:58:28,186 DEBUG [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(1351): Waiting on 408b59e776ffab17a3ee0e8fb465c67c, 408dfc042fd5f2d964a15b01fd44966c, b540ac2e8c4575b424d481848c4b4e52 2024-11-13T05:58:28,194 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408dfc042fd5f2d964a15b01fd44966c/.tmp/colfamily11/8dd36bd4680e463b995a5658f615b9f8 as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408dfc042fd5f2d964a15b01fd44966c/colfamily11/8dd36bd4680e463b995a5658f615b9f8 2024-11-13T05:58:28,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741901_1077 (size=39263) 2024-11-13T05:58:28,200 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408dfc042fd5f2d964a15b01fd44966c/colfamily11/8dd36bd4680e463b995a5658f615b9f8, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T05:58:28,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741901_1077 (size=39263) 2024-11-13T05:58:28,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741901_1077 (size=39263) 2024-11-13T05:58:28,201 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 408dfc042fd5f2d964a15b01fd44966c in 43ms, sequenceid=5, compaction requested=false 2024-11-13T05:58:28,201 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/80710108d60e6f4a1b8975ccfe51e50e/.tmp/colfamily11/f9d54ed6f80140e5b6da3aa118060251 2024-11-13T05:58:28,207 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408dfc042fd5f2d964a15b01fd44966c/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T05:58:28,207 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c. 2024-11-13T05:58:28,207 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 408dfc042fd5f2d964a15b01fd44966c: Waiting for close lock at 1731477508157Running coprocessor pre-close hooks at 1731477508157Disabling compacts and flushes for region at 1731477508157Disabling writes for close at 1731477508158 (+1 ms)Obtaining lock to block concurrent updates at 1731477508158Preparing flush snapshotting stores in 408dfc042fd5f2d964a15b01fd44966c at 1731477508158Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477508158Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c. at 1731477508159 (+1 ms)Flushing 408dfc042fd5f2d964a15b01fd44966c/colfamily11: creating writer at 1731477508159Flushing 408dfc042fd5f2d964a15b01fd44966c/colfamily11: appending metadata at 1731477508175 (+16 ms)Flushing 408dfc042fd5f2d964a15b01fd44966c/colfamily11: closing flushed file at 1731477508175Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3be5ccec: reopening flushed file at 1731477508193 (+18 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 408dfc042fd5f2d964a15b01fd44966c in 43ms, sequenceid=5, compaction requested=false at 1731477508201 (+8 ms)Writing region close event to WAL at 1731477508203 (+2 ms)Running coprocessor post-close hooks at 1731477508207 (+4 ms)Closed at 1731477508207 2024-11-13T05:58:28,207 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731477316849.408dfc042fd5f2d964a15b01fd44966c. 2024-11-13T05:58:28,207 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 408b59e776ffab17a3ee0e8fb465c67c, disabling compactions & flushes 2024-11-13T05:58:28,207 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c. 2024-11-13T05:58:28,208 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c. 2024-11-13T05:58:28,208 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c. after waiting 0 ms 2024-11-13T05:58:28,208 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c. 2024-11-13T05:58:28,208 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 408b59e776ffab17a3ee0e8fb465c67c 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:28,209 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/80710108d60e6f4a1b8975ccfe51e50e/.tmp/colfamily11/f9d54ed6f80140e5b6da3aa118060251 as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/80710108d60e6f4a1b8975ccfe51e50e/colfamily11/f9d54ed6f80140e5b6da3aa118060251 2024-11-13T05:58:28,216 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/80710108d60e6f4a1b8975ccfe51e50e/colfamily11/f9d54ed6f80140e5b6da3aa118060251, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T05:58:28,217 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 80710108d60e6f4a1b8975ccfe51e50e in 50ms, sequenceid=5, compaction requested=false 2024-11-13T05:58:28,222 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/80710108d60e6f4a1b8975ccfe51e50e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T05:58:28,222 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e. 2024-11-13T05:58:28,223 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 80710108d60e6f4a1b8975ccfe51e50e: Waiting for close lock at 1731477508167Running coprocessor pre-close hooks at 1731477508167Disabling compacts and flushes for region at 1731477508167Disabling writes for close at 1731477508167Obtaining lock to block concurrent updates at 1731477508167Preparing flush snapshotting stores in 80710108d60e6f4a1b8975ccfe51e50e at 1731477508167Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477508168 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e. at 1731477508168Flushing 80710108d60e6f4a1b8975ccfe51e50e/colfamily11: creating writer at 1731477508168Flushing 80710108d60e6f4a1b8975ccfe51e50e/colfamily11: appending metadata at 1731477508185 (+17 ms)Flushing 80710108d60e6f4a1b8975ccfe51e50e/colfamily11: closing flushed file at 1731477508186 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21a4a93b: reopening flushed file at 1731477508208 (+22 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 80710108d60e6f4a1b8975ccfe51e50e in 50ms, sequenceid=5, compaction requested=false at 1731477508217 (+9 ms)Writing region close event to WAL at 1731477508218 (+1 ms)Running coprocessor post-close hooks at 1731477508222 (+4 ms)Closed at 1731477508222 2024-11-13T05:58:28,223 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731477316849.80710108d60e6f4a1b8975ccfe51e50e. 2024-11-13T05:58:28,223 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 82c0a84200b3abe2cb346258143147c7, disabling compactions & flushes 2024-11-13T05:58:28,223 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7. 2024-11-13T05:58:28,223 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7. 2024-11-13T05:58:28,223 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7. after waiting 0 ms 2024-11-13T05:58:28,223 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7. 2024-11-13T05:58:28,227 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408b59e776ffab17a3ee0e8fb465c67c/.tmp/colfamily11/2897c7246aa74830b4e15c118dee7ab2 is 52, key is hhh/colfamily11:colfamily11/1731477320554/Put/seqid=0 2024-11-13T05:58:28,229 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/82c0a84200b3abe2cb346258143147c7/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T05:58:28,230 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7. 2024-11-13T05:58:28,230 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 82c0a84200b3abe2cb346258143147c7: Waiting for close lock at 1731477508223Running coprocessor pre-close hooks at 1731477508223Disabling compacts and flushes for region at 1731477508223Disabling writes for close at 1731477508223Writing region close event to WAL at 1731477508225 (+2 ms)Running coprocessor post-close hooks at 1731477508230 (+5 ms)Closed at 1731477508230 2024-11-13T05:58:28,230 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731477316849.82c0a84200b3abe2cb346258143147c7. 2024-11-13T05:58:28,230 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 11abc6d6caf592b351052ae8adf50fd1, disabling compactions & flushes 2024-11-13T05:58:28,230 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1. 2024-11-13T05:58:28,230 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1. 2024-11-13T05:58:28,230 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1. after waiting 0 ms 2024-11-13T05:58:28,230 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1. 2024-11-13T05:58:28,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741902_1078 (size=39263) 2024-11-13T05:58:28,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741902_1078 (size=39263) 2024-11-13T05:58:28,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741902_1078 (size=39263) 2024-11-13T05:58:28,236 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408b59e776ffab17a3ee0e8fb465c67c/.tmp/colfamily11/2897c7246aa74830b4e15c118dee7ab2 2024-11-13T05:58:28,238 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/11abc6d6caf592b351052ae8adf50fd1/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-13T05:58:28,239 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1. 2024-11-13T05:58:28,239 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 11abc6d6caf592b351052ae8adf50fd1: Waiting for close lock at 1731477508230Running coprocessor pre-close hooks at 1731477508230Disabling compacts and flushes for region at 1731477508230Disabling writes for close at 1731477508230Writing region close event to WAL at 1731477508232 (+2 ms)Running coprocessor post-close hooks at 1731477508239 (+7 ms)Closed at 1731477508239 2024-11-13T05:58:28,239 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731477316849.11abc6d6caf592b351052ae8adf50fd1. 2024-11-13T05:58:28,240 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 10c268d6da33b0a4590a478c0f32fd70, disabling compactions & flushes 2024-11-13T05:58:28,240 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70. 2024-11-13T05:58:28,240 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70. 2024-11-13T05:58:28,240 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70. after waiting 0 ms 2024-11-13T05:58:28,240 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70. 2024-11-13T05:58:28,240 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 10c268d6da33b0a4590a478c0f32fd70 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:28,246 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408b59e776ffab17a3ee0e8fb465c67c/.tmp/colfamily11/2897c7246aa74830b4e15c118dee7ab2 as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408b59e776ffab17a3ee0e8fb465c67c/colfamily11/2897c7246aa74830b4e15c118dee7ab2 2024-11-13T05:58:28,252 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408b59e776ffab17a3ee0e8fb465c67c/colfamily11/2897c7246aa74830b4e15c118dee7ab2, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T05:58:28,253 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 408b59e776ffab17a3ee0e8fb465c67c in 45ms, sequenceid=5, compaction requested=false 2024-11-13T05:58:28,257 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408b59e776ffab17a3ee0e8fb465c67c/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T05:58:28,258 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c. 2024-11-13T05:58:28,258 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 408b59e776ffab17a3ee0e8fb465c67c: Waiting for close lock at 1731477508207Running coprocessor pre-close hooks at 1731477508207Disabling compacts and flushes for region at 1731477508207Disabling writes for close at 1731477508208 (+1 ms)Obtaining lock to block concurrent updates at 1731477508208Preparing flush snapshotting stores in 408b59e776ffab17a3ee0e8fb465c67c at 1731477508208Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477508208Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c. at 1731477508209 (+1 ms)Flushing 408b59e776ffab17a3ee0e8fb465c67c/colfamily11: creating writer at 1731477508209Flushing 408b59e776ffab17a3ee0e8fb465c67c/colfamily11: appending metadata at 1731477508227 (+18 ms)Flushing 408b59e776ffab17a3ee0e8fb465c67c/colfamily11: closing flushed file at 1731477508227Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d4883fb: reopening flushed file at 1731477508245 (+18 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 408b59e776ffab17a3ee0e8fb465c67c in 45ms, sequenceid=5, compaction requested=false at 1731477508253 (+8 ms)Writing region close event to WAL at 1731477508254 (+1 ms)Running coprocessor post-close hooks at 1731477508258 (+4 ms)Closed at 1731477508258 2024-11-13T05:58:28,258 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731477316849.408b59e776ffab17a3ee0e8fb465c67c. 2024-11-13T05:58:28,258 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b540ac2e8c4575b424d481848c4b4e52, disabling compactions & flushes 2024-11-13T05:58:28,258 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52. 2024-11-13T05:58:28,258 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52. 2024-11-13T05:58:28,259 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52. after waiting 0 ms 2024-11-13T05:58:28,259 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52. 2024-11-13T05:58:28,259 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing b540ac2e8c4575b424d481848c4b4e52 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-13T05:58:28,260 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10c268d6da33b0a4590a478c0f32fd70/.tmp/colfamily11/8bf5281c406045d3a9fb88fce4fc4104 is 52, key is lll/colfamily11:colfamily11/1731477320847/Put/seqid=0 2024-11-13T05:58:28,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741903_1079 (size=39263) 2024-11-13T05:58:28,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741903_1079 (size=39263) 2024-11-13T05:58:28,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741903_1079 (size=39263) 2024-11-13T05:58:28,268 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10c268d6da33b0a4590a478c0f32fd70/.tmp/colfamily11/8bf5281c406045d3a9fb88fce4fc4104 2024-11-13T05:58:28,275 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10c268d6da33b0a4590a478c0f32fd70/.tmp/colfamily11/8bf5281c406045d3a9fb88fce4fc4104 as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10c268d6da33b0a4590a478c0f32fd70/colfamily11/8bf5281c406045d3a9fb88fce4fc4104 2024-11-13T05:58:28,278 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b540ac2e8c4575b424d481848c4b4e52/.tmp/colfamily11/8a96b3adb2204f83a5469533cc7e68dc is 52, key is kkk/colfamily11:colfamily11/1731477320623/Put/seqid=0 2024-11-13T05:58:28,282 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10c268d6da33b0a4590a478c0f32fd70/colfamily11/8bf5281c406045d3a9fb88fce4fc4104, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T05:58:28,283 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 10c268d6da33b0a4590a478c0f32fd70 in 43ms, sequenceid=5, compaction requested=false 2024-11-13T05:58:28,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741904_1080 (size=39263) 2024-11-13T05:58:28,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741904_1080 (size=39263) 2024-11-13T05:58:28,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741904_1080 (size=39263) 2024-11-13T05:58:28,293 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b540ac2e8c4575b424d481848c4b4e52/.tmp/colfamily11/8a96b3adb2204f83a5469533cc7e68dc 2024-11-13T05:58:28,295 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10c268d6da33b0a4590a478c0f32fd70/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T05:58:28,296 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70. 2024-11-13T05:58:28,296 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 10c268d6da33b0a4590a478c0f32fd70: Waiting for close lock at 1731477508240Running coprocessor pre-close hooks at 1731477508240Disabling compacts and flushes for region at 1731477508240Disabling writes for close at 1731477508240Obtaining lock to block concurrent updates at 1731477508240Preparing flush snapshotting stores in 10c268d6da33b0a4590a478c0f32fd70 at 1731477508240Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477508240Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70. at 1731477508241 (+1 ms)Flushing 10c268d6da33b0a4590a478c0f32fd70/colfamily11: creating writer at 1731477508241Flushing 10c268d6da33b0a4590a478c0f32fd70/colfamily11: appending metadata at 1731477508260 (+19 ms)Flushing 10c268d6da33b0a4590a478c0f32fd70/colfamily11: closing flushed file at 1731477508260Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7384ff93: reopening flushed file at 1731477508274 (+14 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 10c268d6da33b0a4590a478c0f32fd70 in 43ms, sequenceid=5, compaction requested=false at 1731477508283 (+9 ms)Writing region close event to WAL at 1731477508288 (+5 ms)Running coprocessor post-close hooks at 1731477508296 (+8 ms)Closed at 1731477508296 2024-11-13T05:58:28,296 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731477316849.10c268d6da33b0a4590a478c0f32fd70. 2024-11-13T05:58:28,299 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b540ac2e8c4575b424d481848c4b4e52/.tmp/colfamily11/8a96b3adb2204f83a5469533cc7e68dc as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b540ac2e8c4575b424d481848c4b4e52/colfamily11/8a96b3adb2204f83a5469533cc7e68dc 2024-11-13T05:58:28,305 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b540ac2e8c4575b424d481848c4b4e52/colfamily11/8a96b3adb2204f83a5469533cc7e68dc, entries=703, sequenceid=5, filesize=38.3 K 2024-11-13T05:58:28,305 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for b540ac2e8c4575b424d481848c4b4e52 in 46ms, sequenceid=5, compaction requested=false 2024-11-13T05:58:28,310 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b540ac2e8c4575b424d481848c4b4e52/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T05:58:28,310 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52. 2024-11-13T05:58:28,310 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b540ac2e8c4575b424d481848c4b4e52: Waiting for close lock at 1731477508258Running coprocessor pre-close hooks at 1731477508258Disabling compacts and flushes for region at 1731477508258Disabling writes for close at 1731477508259 (+1 ms)Obtaining lock to block concurrent updates at 1731477508259Preparing flush snapshotting stores in b540ac2e8c4575b424d481848c4b4e52 at 1731477508259Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731477508259Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52. at 1731477508260 (+1 ms)Flushing b540ac2e8c4575b424d481848c4b4e52/colfamily11: creating writer at 1731477508260Flushing b540ac2e8c4575b424d481848c4b4e52/colfamily11: appending metadata at 1731477508277 (+17 ms)Flushing b540ac2e8c4575b424d481848c4b4e52/colfamily11: closing flushed file at 1731477508277Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d72ea63: reopening flushed file at 1731477508298 (+21 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for b540ac2e8c4575b424d481848c4b4e52 in 46ms, sequenceid=5, compaction requested=false at 1731477508305 (+7 ms)Writing region close event to WAL at 1731477508306 (+1 ms)Running coprocessor post-close hooks at 1731477508310 (+4 ms)Closed at 1731477508310 2024-11-13T05:58:28,310 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731477316849.b540ac2e8c4575b424d481848c4b4e52. 2024-11-13T05:58:28,386 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(976): stopping server bc386b7bf2fe,40327,1731477314499; all regions closed. 2024-11-13T05:58:28,386 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(976): stopping server bc386b7bf2fe,45383,1731477314448; all regions closed. 2024-11-13T05:58:28,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741865_1041 (size=15722) 2024-11-13T05:58:28,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741865_1041 (size=15722) 2024-11-13T05:58:28,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741835_1011 (size=315396) 2024-11-13T05:58:28,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741865_1041 (size=15722) 2024-11-13T05:58:28,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741835_1011 (size=315396) 2024-11-13T05:58:28,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741835_1011 (size=315396) 2024-11-13T05:58:28,393 DEBUG [RS:2;bc386b7bf2fe:40327 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/oldWALs 2024-11-13T05:58:28,393 DEBUG [RS:1;bc386b7bf2fe:45383 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/oldWALs 2024-11-13T05:58:28,393 INFO [RS:2;bc386b7bf2fe:40327 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bc386b7bf2fe%2C40327%2C1731477314499:(num 1731477316011) 2024-11-13T05:58:28,393 DEBUG [RS:2;bc386b7bf2fe:40327 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:58:28,393 INFO [RS:1;bc386b7bf2fe:45383 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bc386b7bf2fe%2C45383%2C1731477314448.meta:.meta(num 1731477502265) 2024-11-13T05:58:28,393 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:58:28,393 INFO [RS:2;bc386b7bf2fe:40327 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:58:28,394 INFO [RS:2;bc386b7bf2fe:40327 {}] hbase.ChoreService(370): Chore service for: regionserver/bc386b7bf2fe:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T05:58:28,394 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T05:58:28,394 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T05:58:28,394 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T05:58:28,394 INFO [regionserver/bc386b7bf2fe:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:58:28,394 INFO [RS:2;bc386b7bf2fe:40327 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:58:28,394 INFO [RS:2;bc386b7bf2fe:40327 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40327 2024-11-13T05:58:28,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741833_1009 (size=316539) 2024-11-13T05:58:28,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741833_1009 (size=316539) 2024-11-13T05:58:28,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741833_1009 (size=316539) 2024-11-13T05:58:28,400 DEBUG [RS:1;bc386b7bf2fe:45383 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/oldWALs 2024-11-13T05:58:28,401 INFO [RS:1;bc386b7bf2fe:45383 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bc386b7bf2fe%2C45383%2C1731477314448:(num 1731477316011) 2024-11-13T05:58:28,401 DEBUG [RS:1;bc386b7bf2fe:45383 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:58:28,401 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:58:28,401 INFO [RS:1;bc386b7bf2fe:45383 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:58:28,401 INFO [RS:1;bc386b7bf2fe:45383 {}] hbase.ChoreService(370): Chore service for: regionserver/bc386b7bf2fe:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T05:58:28,401 INFO [RS:1;bc386b7bf2fe:45383 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:58:28,401 INFO [regionserver/bc386b7bf2fe:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:58:28,401 INFO [RS:1;bc386b7bf2fe:45383 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45383 2024-11-13T05:58:28,455 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bc386b7bf2fe,40327,1731477314499 2024-11-13T05:58:28,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:58:28,455 INFO [RS:2;bc386b7bf2fe:40327 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:58:28,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bc386b7bf2fe,45383,1731477314448 2024-11-13T05:58:28,464 INFO [RS:1;bc386b7bf2fe:45383 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:58:28,464 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bc386b7bf2fe,45383,1731477314448] 2024-11-13T05:58:28,480 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bc386b7bf2fe,45383,1731477314448 already deleted, retry=false 2024-11-13T05:58:28,481 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bc386b7bf2fe,45383,1731477314448 expired; onlineServers=1 2024-11-13T05:58:28,481 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bc386b7bf2fe,40327,1731477314499] 2024-11-13T05:58:28,488 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bc386b7bf2fe,40327,1731477314499 already deleted, retry=false 2024-11-13T05:58:28,489 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bc386b7bf2fe,40327,1731477314499 expired; onlineServers=0 2024-11-13T05:58:28,489 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'bc386b7bf2fe,37807,1731477313603' ***** 2024-11-13T05:58:28,489 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T05:58:28,489 INFO [M:0;bc386b7bf2fe:37807 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:58:28,489 INFO [M:0;bc386b7bf2fe:37807 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:58:28,489 DEBUG [M:0;bc386b7bf2fe:37807 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T05:58:28,489 DEBUG [M:0;bc386b7bf2fe:37807 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T05:58:28,490 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T05:58:28,490 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.large.0-1731477315596 {}] cleaner.HFileCleaner(306): Exit Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.large.0-1731477315596,5,FailOnTimeoutGroup] 2024-11-13T05:58:28,490 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.small.0-1731477315599 {}] cleaner.HFileCleaner(306): Exit Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.small.0-1731477315599,5,FailOnTimeoutGroup] 2024-11-13T05:58:28,490 INFO [M:0;bc386b7bf2fe:37807 {}] hbase.ChoreService(370): Chore service for: master/bc386b7bf2fe:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T05:58:28,490 INFO [M:0;bc386b7bf2fe:37807 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:58:28,491 DEBUG [M:0;bc386b7bf2fe:37807 {}] master.HMaster(1795): Stopping service threads 2024-11-13T05:58:28,491 INFO [M:0;bc386b7bf2fe:37807 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T05:58:28,491 INFO [M:0;bc386b7bf2fe:37807 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:58:28,492 INFO [M:0;bc386b7bf2fe:37807 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T05:58:28,492 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T05:58:28,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T05:58:28,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:58:28,497 DEBUG [M:0;bc386b7bf2fe:37807 {}] zookeeper.ZKUtil(347): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T05:58:28,498 WARN [M:0;bc386b7bf2fe:37807 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T05:58:28,499 INFO [M:0;bc386b7bf2fe:37807 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/.lastflushedseqids 2024-11-13T05:58:28,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741905_1081 (size=2837) 2024-11-13T05:58:28,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741905_1081 (size=2837) 2024-11-13T05:58:28,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741905_1081 (size=2837) 2024-11-13T05:58:28,511 INFO [M:0;bc386b7bf2fe:37807 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T05:58:28,511 INFO [M:0;bc386b7bf2fe:37807 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T05:58:28,511 DEBUG [M:0;bc386b7bf2fe:37807 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:58:28,511 INFO [M:0;bc386b7bf2fe:37807 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:58:28,511 DEBUG [M:0;bc386b7bf2fe:37807 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:58:28,511 DEBUG [M:0;bc386b7bf2fe:37807 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:58:28,511 DEBUG [M:0;bc386b7bf2fe:37807 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:58:28,511 INFO [M:0;bc386b7bf2fe:37807 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=344.56 KB heapSize=404.96 KB 2024-11-13T05:58:28,527 DEBUG [M:0;bc386b7bf2fe:37807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7b984cbe27964a6486c0150be9af979c is 82, key is hbase:meta,,1/info:regioninfo/1731477502583/Put/seqid=0 2024-11-13T05:58:28,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741906_1082 (size=6071) 2024-11-13T05:58:28,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741906_1082 (size=6071) 2024-11-13T05:58:28,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741906_1082 (size=6071) 2024-11-13T05:58:28,534 INFO [M:0;bc386b7bf2fe:37807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7b984cbe27964a6486c0150be9af979c 2024-11-13T05:58:28,555 DEBUG [M:0;bc386b7bf2fe:37807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/71dbdd9b499447a4af93585278f89e73 is 4105, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731477318077/Put/seqid=0 2024-11-13T05:58:28,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741907_1083 (size=13502) 2024-11-13T05:58:28,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741907_1083 (size=13502) 2024-11-13T05:58:28,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741907_1083 (size=13502) 2024-11-13T05:58:28,566 INFO [M:0;bc386b7bf2fe:37807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=343.33 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/71dbdd9b499447a4af93585278f89e73 2024-11-13T05:58:28,572 INFO [RS:2;bc386b7bf2fe:40327 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:58:28,572 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:58:28,572 INFO [RS:2;bc386b7bf2fe:40327 {}] regionserver.HRegionServer(1031): Exiting; stopping=bc386b7bf2fe,40327,1731477314499; zookeeper connection closed. 2024-11-13T05:58:28,572 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40327-0x101340ca7aa0003, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:58:28,573 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2339b384 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2339b384 2024-11-13T05:58:28,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:58:28,580 INFO [RS:1;bc386b7bf2fe:45383 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:58:28,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45383-0x101340ca7aa0002, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:58:28,581 INFO [RS:1;bc386b7bf2fe:45383 {}] regionserver.HRegionServer(1031): Exiting; stopping=bc386b7bf2fe,45383,1731477314448; zookeeper connection closed. 2024-11-13T05:58:28,581 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@60c4492a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@60c4492a 2024-11-13T05:58:28,581 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-13T05:58:28,590 DEBUG [M:0;bc386b7bf2fe:37807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c8ad9ec5d5d34003981088b842f645e4 is 69, key is bc386b7bf2fe,40327,1731477314499/rs:state/1731477315694/Put/seqid=0 2024-11-13T05:58:28,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741908_1084 (size=5442) 2024-11-13T05:58:28,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741908_1084 (size=5442) 2024-11-13T05:58:28,597 INFO [M:0;bc386b7bf2fe:37807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c8ad9ec5d5d34003981088b842f645e4 2024-11-13T05:58:28,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741908_1084 (size=5442) 2024-11-13T05:58:28,602 INFO [M:0;bc386b7bf2fe:37807 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c8ad9ec5d5d34003981088b842f645e4 2024-11-13T05:58:28,603 DEBUG [M:0;bc386b7bf2fe:37807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7b984cbe27964a6486c0150be9af979c as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7b984cbe27964a6486c0150be9af979c 2024-11-13T05:58:28,609 INFO [M:0;bc386b7bf2fe:37807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7b984cbe27964a6486c0150be9af979c, entries=14, sequenceid=635, filesize=5.9 K 2024-11-13T05:58:28,610 DEBUG [M:0;bc386b7bf2fe:37807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/71dbdd9b499447a4af93585278f89e73 as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/71dbdd9b499447a4af93585278f89e73 2024-11-13T05:58:28,617 INFO [M:0;bc386b7bf2fe:37807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/71dbdd9b499447a4af93585278f89e73, entries=81, sequenceid=635, filesize=13.2 K 2024-11-13T05:58:28,618 DEBUG [M:0;bc386b7bf2fe:37807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c8ad9ec5d5d34003981088b842f645e4 as hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c8ad9ec5d5d34003981088b842f645e4 2024-11-13T05:58:28,624 INFO [M:0;bc386b7bf2fe:37807 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c8ad9ec5d5d34003981088b842f645e4 2024-11-13T05:58:28,624 INFO [M:0;bc386b7bf2fe:37807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37737/user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c8ad9ec5d5d34003981088b842f645e4, entries=3, sequenceid=635, filesize=5.3 K 2024-11-13T05:58:28,625 INFO [M:0;bc386b7bf2fe:37807 {}] regionserver.HRegion(3140): Finished flush of dataSize ~344.56 KB/352832, heapSize ~404.66 KB/414376, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 114ms, sequenceid=635, compaction requested=false 2024-11-13T05:58:28,626 INFO [M:0;bc386b7bf2fe:37807 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:58:28,626 DEBUG [M:0;bc386b7bf2fe:37807 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731477508511Disabling compacts and flushes for region at 1731477508511Disabling writes for close at 1731477508511Obtaining lock to block concurrent updates at 1731477508511Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731477508511Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=352832, getHeapSize=414616, getOffHeapSize=0, getCellsCount=800 at 1731477508512 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731477508512Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731477508512Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731477508527 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731477508527Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731477508539 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731477508555 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731477508555Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731477508570 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731477508589 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731477508589Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7198aea: reopening flushed file at 1731477508602 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b121f2a: reopening flushed file at 1731477508609 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e8a5d61: reopening flushed file at 1731477508617 (+8 ms)Finished flush of dataSize ~344.56 KB/352832, heapSize ~404.66 KB/414376, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 114ms, sequenceid=635, compaction requested=false at 1731477508625 (+8 ms)Writing region close event to WAL at 1731477508626 (+1 ms)Closed at 1731477508626 2024-11-13T05:58:28,630 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/bb18aff5-da07-b2a5-df17-340018b2b455/MasterData/WALs/bc386b7bf2fe,37807,1731477313603/bc386b7bf2fe%2C37807%2C1731477313603.1731477314927 not finished, retry = 0 2024-11-13T05:58:28,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35961 is added to blk_1073741830_1006 (size=395186) 2024-11-13T05:58:28,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41907 is added to blk_1073741830_1006 (size=395186) 2024-11-13T05:58:28,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37319 is added to blk_1073741830_1006 (size=395186) 2024-11-13T05:58:28,731 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:58:28,731 INFO [M:0;bc386b7bf2fe:37807 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T05:58:28,731 INFO [M:0;bc386b7bf2fe:37807 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37807 2024-11-13T05:58:28,731 INFO [M:0;bc386b7bf2fe:37807 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:58:28,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:58:28,855 INFO [M:0;bc386b7bf2fe:37807 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:58:28,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37807-0x101340ca7aa0000, quorum=127.0.0.1:58994, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:58:28,899 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@22333f25{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:58:28,905 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@eec57c1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:58:28,905 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:58:28,906 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@17888407{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:58:28,906 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@30067af9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/hadoop.log.dir/,STOPPED} 2024-11-13T05:58:28,908 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:58:28,908 WARN [BP-1769423020-172.17.0.2-1731477309687 heartbeating to localhost/127.0.0.1:37737 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:58:28,908 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:58:28,908 WARN [BP-1769423020-172.17.0.2-1731477309687 heartbeating to localhost/127.0.0.1:37737 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1769423020-172.17.0.2-1731477309687 (Datanode Uuid 353a8d27-bc02-479e-af6a-5157f17bcd56) service to localhost/127.0.0.1:37737 2024-11-13T05:58:28,910 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/cluster_231b4c26-e46c-eae9-a748-e1bc31753d3a/data/data5/current/BP-1769423020-172.17.0.2-1731477309687 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:58:28,910 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/cluster_231b4c26-e46c-eae9-a748-e1bc31753d3a/data/data6/current/BP-1769423020-172.17.0.2-1731477309687 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:58:28,911 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:58:28,912 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1ba455be{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:58:28,913 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@259771a6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:58:28,913 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:58:28,913 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6c516a28{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:58:28,913 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f4017e5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/hadoop.log.dir/,STOPPED} 2024-11-13T05:58:28,914 WARN [BP-1769423020-172.17.0.2-1731477309687 heartbeating to localhost/127.0.0.1:37737 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:58:28,914 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:58:28,914 WARN [BP-1769423020-172.17.0.2-1731477309687 heartbeating to localhost/127.0.0.1:37737 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1769423020-172.17.0.2-1731477309687 (Datanode Uuid 50dd1970-68b2-42ac-ad88-d3fbf24e3d13) service to localhost/127.0.0.1:37737 2024-11-13T05:58:28,914 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:58:28,915 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/cluster_231b4c26-e46c-eae9-a748-e1bc31753d3a/data/data3/current/BP-1769423020-172.17.0.2-1731477309687 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:58:28,915 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/cluster_231b4c26-e46c-eae9-a748-e1bc31753d3a/data/data4/current/BP-1769423020-172.17.0.2-1731477309687 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:58:28,915 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:58:28,917 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4d81bf0e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T05:58:28,917 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@572fe942{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:58:28,917 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:58:28,917 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@664a7783{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:58:28,918 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7da31712{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/hadoop.log.dir/,STOPPED} 2024-11-13T05:58:28,918 WARN [BP-1769423020-172.17.0.2-1731477309687 heartbeating to localhost/127.0.0.1:37737 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T05:58:28,918 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T05:58:28,918 WARN [BP-1769423020-172.17.0.2-1731477309687 heartbeating to localhost/127.0.0.1:37737 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1769423020-172.17.0.2-1731477309687 (Datanode Uuid 999c1d0c-5cd8-4a6b-b1fa-b5872fc5c220) service to localhost/127.0.0.1:37737 2024-11-13T05:58:28,919 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T05:58:28,919 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/cluster_231b4c26-e46c-eae9-a748-e1bc31753d3a/data/data1/current/BP-1769423020-172.17.0.2-1731477309687 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:58:28,919 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/cluster_231b4c26-e46c-eae9-a748-e1bc31753d3a/data/data2/current/BP-1769423020-172.17.0.2-1731477309687 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T05:58:28,920 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T05:58:28,931 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5183f126{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T05:58:28,932 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4350931a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T05:58:28,932 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T05:58:28,932 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4974766f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T05:58:28,933 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2c447bb0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f640a604-7fb1-3e42-74c0-778936a4f21f/hadoop.log.dir/,STOPPED} 2024-11-13T05:58:28,947 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T05:58:29,019 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-13T05:58:29,029 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestSCPWithMeta#testRecoveryAndDoubleExecutionOnRsWithMeta Thread=93 (was 11) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/bc386b7bf2fe:0.Chore.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37737 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@414f4765 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37737 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37737 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (102774891) connection to localhost/127.0.0.1:37737 from jenkins.hfs.1 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:37737 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: nioEventLoopGroup-6-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-7-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: IPC Client (102774891) connection to localhost/127.0.0.1:37737 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37737 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/bc386b7bf2fe:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: LeaseRenewer:jenkins@localhost:37737 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: nioEventLoopGroup-7-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/bc386b7bf2fe:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: regionserver/bc386b7bf2fe:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: regionserver/bc386b7bf2fe:0.logRoller java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180) Potentially hanging thread: master/bc386b7bf2fe:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:37737 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/bc386b7bf2fe:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: BootstrapNodeManager java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (102774891) connection to localhost/127.0.0.1:37737 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-7-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: region-location-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: IPC Client (102774891) connection to localhost/127.0.0.1:37737 from jenkins.hfs.2 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=456 (was 285) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=86 (was 280), ProcessCount=11 (was 11), AvailableMemoryMB=6916 (was 8320)