2024-11-08 13:28:40,394 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-08 13:28:40,414 main DEBUG Took 0.017234 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 13:28:40,414 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 13:28:40,415 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 13:28:40,416 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 13:28:40,417 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:28:40,426 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 13:28:40,443 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:28:40,444 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:28:40,444 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:28:40,445 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:28:40,445 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:28:40,446 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:28:40,447 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:28:40,447 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:28:40,447 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:28:40,448 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:28:40,448 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:28:40,449 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:28:40,449 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:28:40,449 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:28:40,450 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:28:40,450 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:28:40,451 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:28:40,451 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:28:40,451 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:28:40,452 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:28:40,452 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:28:40,452 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:28:40,453 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:28:40,453 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:28:40,453 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:28:40,454 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 13:28:40,455 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:28:40,456 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 13:28:40,458 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 13:28:40,458 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 13:28:40,459 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 13:28:40,460 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 13:28:40,467 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 13:28:40,469 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 13:28:40,471 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 13:28:40,471 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 13:28:40,472 main DEBUG createAppenders(={Console}) 2024-11-08 13:28:40,472 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-08 13:28:40,472 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-08 13:28:40,473 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-08 13:28:40,473 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 13:28:40,474 main DEBUG OutputStream closed 2024-11-08 13:28:40,474 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 13:28:40,474 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 13:28:40,474 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-08 13:28:40,540 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 13:28:40,542 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 13:28:40,543 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 13:28:40,544 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 13:28:40,544 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 13:28:40,544 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 13:28:40,545 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 13:28:40,545 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 13:28:40,545 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 13:28:40,546 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 13:28:40,546 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 13:28:40,546 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 13:28:40,546 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 13:28:40,547 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 13:28:40,547 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 13:28:40,547 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 13:28:40,548 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 13:28:40,548 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 13:28:40,550 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 13:28:40,550 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-08 13:28:40,551 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 13:28:40,551 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-08T13:28:40,563 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta timeout: 13 mins 2024-11-08 13:28:40,566 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 13:28:40,566 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T13:28:40,600 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestSCPWithMeta#testRecoveryAndDoubleExecutionOnRsWithMeta Thread=11, OpenFileDescriptor=285, MaxFileDescriptor=1048576, SystemLoadAverage=100, ProcessCount=11, AvailableMemoryMB=3896 2024-11-08T13:28:40,834 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff 2024-11-08T13:28:40,835 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-08T13:28:40,855 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/cluster_fdfcb37c-5278-989a-1d0a-cd8c890d5157, deleteOnExit=true 2024-11-08T13:28:40,855 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-08T13:28:40,857 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/test.cache.data in system properties and HBase conf 2024-11-08T13:28:40,857 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/hadoop.tmp.dir in system properties and HBase conf 2024-11-08T13:28:40,858 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/hadoop.log.dir in system properties and HBase conf 2024-11-08T13:28:40,859 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-08T13:28:40,860 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-08T13:28:40,860 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-08T13:28:40,937 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T13:28:41,022 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:28:41,026 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-08T13:28:41,027 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-08T13:28:41,028 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-08T13:28:41,028 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T13:28:41,029 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-08T13:28:41,030 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-08T13:28:41,030 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T13:28:41,031 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T13:28:41,031 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-08T13:28:41,032 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/nfs.dump.dir in system properties and HBase conf 2024-11-08T13:28:41,033 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/java.io.tmpdir in system properties and HBase conf 2024-11-08T13:28:41,034 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T13:28:41,034 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-08T13:28:41,035 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-08T13:28:41,821 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-08T13:28:41,898 INFO [Time-limited test {}] log.Log(170): Logging initialized @2179ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-08T13:28:41,964 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T13:28:42,031 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T13:28:42,054 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T13:28:42,055 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T13:28:42,056 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-08T13:28:42,067 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T13:28:42,071 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@58897554{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/hadoop.log.dir/,AVAILABLE} 2024-11-08T13:28:42,072 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a372edd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T13:28:42,238 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1aa62678{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/java.io.tmpdir/jetty-localhost-33313-hadoop-hdfs-3_4_1-tests_jar-_-any-13484247778008267742/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T13:28:42,244 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1c0c42ad{HTTP/1.1, (http/1.1)}{localhost:33313} 2024-11-08T13:28:42,245 INFO [Time-limited test {}] server.Server(415): Started @2526ms 2024-11-08T13:28:42,566 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T13:28:42,575 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T13:28:42,576 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T13:28:42,577 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T13:28:42,577 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-08T13:28:42,578 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b71ffd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/hadoop.log.dir/,AVAILABLE} 2024-11-08T13:28:42,578 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@63d82da9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T13:28:42,692 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4255463d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/java.io.tmpdir/jetty-localhost-43247-hadoop-hdfs-3_4_1-tests_jar-_-any-14676082204306385233/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T13:28:42,693 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5e48d68a{HTTP/1.1, (http/1.1)}{localhost:43247} 2024-11-08T13:28:42,693 INFO [Time-limited test {}] server.Server(415): Started @2975ms 2024-11-08T13:28:42,745 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T13:28:42,875 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T13:28:42,883 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T13:28:42,894 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T13:28:42,894 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T13:28:42,895 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T13:28:42,895 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@63779849{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/hadoop.log.dir/,AVAILABLE} 2024-11-08T13:28:42,896 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a292a59{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T13:28:43,030 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3635c31e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/java.io.tmpdir/jetty-localhost-40473-hadoop-hdfs-3_4_1-tests_jar-_-any-9589950422185052220/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T13:28:43,031 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@78839a64{HTTP/1.1, (http/1.1)}{localhost:40473} 2024-11-08T13:28:43,031 INFO [Time-limited test {}] server.Server(415): Started @3312ms 2024-11-08T13:28:43,033 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T13:28:43,079 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T13:28:43,088 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T13:28:43,091 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T13:28:43,092 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T13:28:43,092 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T13:28:43,093 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ed2891b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/hadoop.log.dir/,AVAILABLE} 2024-11-08T13:28:43,094 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@43bf9dc1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T13:28:43,117 WARN [Thread-103 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/cluster_fdfcb37c-5278-989a-1d0a-cd8c890d5157/data/data1/current/BP-1581590160-172.17.0.2-1731072521610/current, will proceed with Du for space computation calculation, 2024-11-08T13:28:43,117 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/cluster_fdfcb37c-5278-989a-1d0a-cd8c890d5157/data/data2/current/BP-1581590160-172.17.0.2-1731072521610/current, will proceed with Du for space computation calculation, 2024-11-08T13:28:43,120 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/cluster_fdfcb37c-5278-989a-1d0a-cd8c890d5157/data/data3/current/BP-1581590160-172.17.0.2-1731072521610/current, will proceed with Du for space computation calculation, 2024-11-08T13:28:43,121 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/cluster_fdfcb37c-5278-989a-1d0a-cd8c890d5157/data/data4/current/BP-1581590160-172.17.0.2-1731072521610/current, will proceed with Du for space computation calculation, 2024-11-08T13:28:43,154 WARN [Thread-83 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T13:28:43,154 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T13:28:43,209 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@486c2677{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/java.io.tmpdir/jetty-localhost-35581-hadoop-hdfs-3_4_1-tests_jar-_-any-13727180801490433380/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T13:28:43,210 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6738d157{HTTP/1.1, (http/1.1)}{localhost:35581} 2024-11-08T13:28:43,210 INFO [Time-limited test {}] server.Server(415): Started @3492ms 2024-11-08T13:28:43,212 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T13:28:43,214 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf4be62672ccce05f with lease ID 0xac645642010d75b: Processing first storage report for DS-d3e6e3df-bbf9-4750-9cd5-a9cb825eda4a from datanode DatanodeRegistration(127.0.0.1:36145, datanodeUuid=66093ed1-7a90-4dc7-ab91-0b1aff75b3f1, infoPort=41369, infoSecurePort=0, ipcPort=44995, storageInfo=lv=-57;cid=testClusterID;nsid=1668058525;c=1731072521610) 2024-11-08T13:28:43,215 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf4be62672ccce05f with lease ID 0xac645642010d75b: from storage DS-d3e6e3df-bbf9-4750-9cd5-a9cb825eda4a node DatanodeRegistration(127.0.0.1:36145, datanodeUuid=66093ed1-7a90-4dc7-ab91-0b1aff75b3f1, infoPort=41369, infoSecurePort=0, ipcPort=44995, storageInfo=lv=-57;cid=testClusterID;nsid=1668058525;c=1731072521610), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-08T13:28:43,215 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x432f1beb8ceb27fa with lease ID 0xac645642010d75a: Processing first storage report for DS-067499be-1a0c-4eb6-83cf-12ed63a0b51c from datanode DatanodeRegistration(127.0.0.1:36805, datanodeUuid=ec5066ce-adb8-4b4d-bbb6-cb0173e8c3bc, infoPort=44267, infoSecurePort=0, ipcPort=41705, storageInfo=lv=-57;cid=testClusterID;nsid=1668058525;c=1731072521610) 2024-11-08T13:28:43,216 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x432f1beb8ceb27fa with lease ID 0xac645642010d75a: from storage DS-067499be-1a0c-4eb6-83cf-12ed63a0b51c node DatanodeRegistration(127.0.0.1:36805, datanodeUuid=ec5066ce-adb8-4b4d-bbb6-cb0173e8c3bc, infoPort=44267, infoSecurePort=0, ipcPort=41705, storageInfo=lv=-57;cid=testClusterID;nsid=1668058525;c=1731072521610), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T13:28:43,216 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf4be62672ccce05f with lease ID 0xac645642010d75b: Processing first storage report for DS-d224248b-a1ba-450d-9bfb-040b592dd676 from datanode DatanodeRegistration(127.0.0.1:36145, datanodeUuid=66093ed1-7a90-4dc7-ab91-0b1aff75b3f1, infoPort=41369, infoSecurePort=0, ipcPort=44995, storageInfo=lv=-57;cid=testClusterID;nsid=1668058525;c=1731072521610) 2024-11-08T13:28:43,216 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf4be62672ccce05f with lease ID 0xac645642010d75b: from storage DS-d224248b-a1ba-450d-9bfb-040b592dd676 node DatanodeRegistration(127.0.0.1:36145, datanodeUuid=66093ed1-7a90-4dc7-ab91-0b1aff75b3f1, infoPort=41369, infoSecurePort=0, ipcPort=44995, storageInfo=lv=-57;cid=testClusterID;nsid=1668058525;c=1731072521610), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T13:28:43,216 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x432f1beb8ceb27fa with lease ID 0xac645642010d75a: Processing first storage report for DS-1aa0d3da-d31c-490b-bfb4-78df9cff4243 from datanode DatanodeRegistration(127.0.0.1:36805, datanodeUuid=ec5066ce-adb8-4b4d-bbb6-cb0173e8c3bc, infoPort=44267, infoSecurePort=0, ipcPort=41705, storageInfo=lv=-57;cid=testClusterID;nsid=1668058525;c=1731072521610) 2024-11-08T13:28:43,216 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x432f1beb8ceb27fa with lease ID 0xac645642010d75a: from storage DS-1aa0d3da-d31c-490b-bfb4-78df9cff4243 node DatanodeRegistration(127.0.0.1:36805, datanodeUuid=ec5066ce-adb8-4b4d-bbb6-cb0173e8c3bc, infoPort=44267, infoSecurePort=0, ipcPort=41705, storageInfo=lv=-57;cid=testClusterID;nsid=1668058525;c=1731072521610), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T13:28:43,305 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/cluster_fdfcb37c-5278-989a-1d0a-cd8c890d5157/data/data5/current/BP-1581590160-172.17.0.2-1731072521610/current, will proceed with Du for space computation calculation, 2024-11-08T13:28:43,307 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/cluster_fdfcb37c-5278-989a-1d0a-cd8c890d5157/data/data6/current/BP-1581590160-172.17.0.2-1731072521610/current, will proceed with Du for space computation calculation, 2024-11-08T13:28:43,332 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T13:28:43,338 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x533bb78452ca52da with lease ID 0xac645642010d75c: Processing first storage report for DS-4c23f8af-3e26-4caf-af50-9edfbeedfc0d from datanode DatanodeRegistration(127.0.0.1:38513, datanodeUuid=ccec3428-e338-4216-a1fe-74212cc76f25, infoPort=43697, infoSecurePort=0, ipcPort=42037, storageInfo=lv=-57;cid=testClusterID;nsid=1668058525;c=1731072521610) 2024-11-08T13:28:43,339 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x533bb78452ca52da with lease ID 0xac645642010d75c: from storage DS-4c23f8af-3e26-4caf-af50-9edfbeedfc0d node DatanodeRegistration(127.0.0.1:38513, datanodeUuid=ccec3428-e338-4216-a1fe-74212cc76f25, infoPort=43697, infoSecurePort=0, ipcPort=42037, storageInfo=lv=-57;cid=testClusterID;nsid=1668058525;c=1731072521610), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T13:28:43,339 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x533bb78452ca52da with lease ID 0xac645642010d75c: Processing first storage report for DS-aed921c9-e210-462e-915c-157a5cf6cbf6 from datanode DatanodeRegistration(127.0.0.1:38513, datanodeUuid=ccec3428-e338-4216-a1fe-74212cc76f25, infoPort=43697, infoSecurePort=0, ipcPort=42037, storageInfo=lv=-57;cid=testClusterID;nsid=1668058525;c=1731072521610) 2024-11-08T13:28:43,339 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x533bb78452ca52da with lease ID 0xac645642010d75c: from storage DS-aed921c9-e210-462e-915c-157a5cf6cbf6 node DatanodeRegistration(127.0.0.1:38513, datanodeUuid=ccec3428-e338-4216-a1fe-74212cc76f25, infoPort=43697, infoSecurePort=0, ipcPort=42037, storageInfo=lv=-57;cid=testClusterID;nsid=1668058525;c=1731072521610), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T13:28:43,522 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff 2024-11-08T13:28:43,588 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/cluster_fdfcb37c-5278-989a-1d0a-cd8c890d5157/zookeeper_0, clientPort=50677, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/cluster_fdfcb37c-5278-989a-1d0a-cd8c890d5157/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/cluster_fdfcb37c-5278-989a-1d0a-cd8c890d5157/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T13:28:43,597 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50677 2024-11-08T13:28:43,607 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T13:28:43,610 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T13:28:43,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741825_1001 (size=7) 2024-11-08T13:28:43,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741825_1001 (size=7) 2024-11-08T13:28:43,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741825_1001 (size=7) 2024-11-08T13:28:44,237 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f with version=8 2024-11-08T13:28:44,238 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/hbase-staging 2024-11-08T13:28:44,323 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T13:28:44,510 INFO [Time-limited test {}] client.ConnectionUtils(128): master/6ccfea20c990:0 server-side Connection retries=45 2024-11-08T13:28:44,518 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:28:44,518 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T13:28:44,522 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T13:28:44,522 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:28:44,522 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T13:28:44,646 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T13:28:44,700 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T13:28:44,708 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T13:28:44,712 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T13:28:44,733 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 35313 (auto-detected) 2024-11-08T13:28:44,734 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-08T13:28:44,750 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40009 2024-11-08T13:28:44,769 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40009 connecting to ZooKeeper ensemble=127.0.0.1:50677 2024-11-08T13:28:44,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:400090x0, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T13:28:44,798 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40009-0x1001f85d9a10000 connected 2024-11-08T13:28:44,824 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T13:28:44,826 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T13:28:44,834 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:28:44,838 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f, hbase.cluster.distributed=false 2024-11-08T13:28:44,860 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T13:28:44,864 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40009 2024-11-08T13:28:44,865 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40009 2024-11-08T13:28:44,865 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40009 2024-11-08T13:28:44,866 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40009 2024-11-08T13:28:44,867 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40009 2024-11-08T13:28:44,957 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6ccfea20c990:0 server-side Connection retries=45 2024-11-08T13:28:44,959 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:28:44,959 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T13:28:44,959 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T13:28:44,959 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:28:44,959 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T13:28:44,962 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T13:28:44,964 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T13:28:44,965 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39565 2024-11-08T13:28:44,967 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39565 connecting to ZooKeeper ensemble=127.0.0.1:50677 2024-11-08T13:28:44,968 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T13:28:44,970 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T13:28:44,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:395650x0, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T13:28:44,976 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:395650x0, quorum=127.0.0.1:50677, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:28:44,976 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39565-0x1001f85d9a10001 connected 2024-11-08T13:28:44,980 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T13:28:44,987 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T13:28:44,989 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39565-0x1001f85d9a10001, quorum=127.0.0.1:50677, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T13:28:44,995 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39565-0x1001f85d9a10001, quorum=127.0.0.1:50677, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T13:28:44,995 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39565 2024-11-08T13:28:44,996 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39565 2024-11-08T13:28:44,996 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39565 2024-11-08T13:28:44,997 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39565 2024-11-08T13:28:44,997 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39565 2024-11-08T13:28:45,013 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6ccfea20c990:0 server-side Connection retries=45 2024-11-08T13:28:45,013 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:28:45,013 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T13:28:45,014 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T13:28:45,014 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:28:45,014 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T13:28:45,014 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T13:28:45,014 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T13:28:45,016 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35061 2024-11-08T13:28:45,017 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35061 connecting to ZooKeeper ensemble=127.0.0.1:50677 2024-11-08T13:28:45,018 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T13:28:45,021 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T13:28:45,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:350610x0, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T13:28:45,027 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35061-0x1001f85d9a10002 connected 2024-11-08T13:28:45,027 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:28:45,027 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T13:28:45,028 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T13:28:45,029 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T13:28:45,031 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T13:28:45,033 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35061 2024-11-08T13:28:45,033 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35061 2024-11-08T13:28:45,036 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35061 2024-11-08T13:28:45,036 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35061 2024-11-08T13:28:45,037 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35061 2024-11-08T13:28:45,052 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6ccfea20c990:0 server-side Connection retries=45 2024-11-08T13:28:45,052 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:28:45,052 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T13:28:45,053 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T13:28:45,053 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:28:45,053 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T13:28:45,053 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T13:28:45,053 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T13:28:45,054 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35367 2024-11-08T13:28:45,056 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35367 connecting to ZooKeeper ensemble=127.0.0.1:50677 2024-11-08T13:28:45,057 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T13:28:45,059 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T13:28:45,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:353670x0, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T13:28:45,065 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:353670x0, quorum=127.0.0.1:50677, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:28:45,066 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35367-0x1001f85d9a10003 connected 2024-11-08T13:28:45,066 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T13:28:45,067 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T13:28:45,069 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T13:28:45,070 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T13:28:45,072 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35367 2024-11-08T13:28:45,073 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35367 2024-11-08T13:28:45,073 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35367 2024-11-08T13:28:45,076 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35367 2024-11-08T13:28:45,076 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35367 2024-11-08T13:28:45,094 DEBUG [M:0;6ccfea20c990:40009 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;6ccfea20c990:40009 2024-11-08T13:28:45,095 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/6ccfea20c990,40009,1731072524365 2024-11-08T13:28:45,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:28:45,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:28:45,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:28:45,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39565-0x1001f85d9a10001, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:28:45,103 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/6ccfea20c990,40009,1731072524365 2024-11-08T13:28:45,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T13:28:45,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39565-0x1001f85d9a10001, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T13:28:45,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T13:28:45,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:45,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39565-0x1001f85d9a10001, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:45,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:45,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:45,126 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T13:28:45,127 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/6ccfea20c990,40009,1731072524365 from backup master directory 2024-11-08T13:28:45,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:28:45,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39565-0x1001f85d9a10001, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:28:45,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:28:45,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/6ccfea20c990,40009,1731072524365 2024-11-08T13:28:45,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:28:45,130 WARN [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T13:28:45,130 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=6ccfea20c990,40009,1731072524365 2024-11-08T13:28:45,133 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-08T13:28:45,134 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-08T13:28:45,185 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/hbase.id] with ID: 726d013e-8c45-4909-bccd-5b44ea40be15 2024-11-08T13:28:45,185 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/.tmp/hbase.id 2024-11-08T13:28:45,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741826_1002 (size=42) 2024-11-08T13:28:45,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741826_1002 (size=42) 2024-11-08T13:28:45,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741826_1002 (size=42) 2024-11-08T13:28:45,202 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/.tmp/hbase.id]:[hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/hbase.id] 2024-11-08T13:28:45,248 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T13:28:45,253 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T13:28:45,271 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-08T13:28:45,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39565-0x1001f85d9a10001, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:45,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:45,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:45,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:45,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741827_1003 (size=196) 2024-11-08T13:28:45,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741827_1003 (size=196) 2024-11-08T13:28:45,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741827_1003 (size=196) 2024-11-08T13:28:45,309 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T13:28:45,312 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T13:28:45,326 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:28:45,330 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T13:28:45,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741828_1004 (size=1189) 2024-11-08T13:28:45,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741828_1004 (size=1189) 2024-11-08T13:28:45,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741828_1004 (size=1189) 2024-11-08T13:28:45,380 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/data/master/store 2024-11-08T13:28:45,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741829_1005 (size=34) 2024-11-08T13:28:45,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741829_1005 (size=34) 2024-11-08T13:28:45,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741829_1005 (size=34) 2024-11-08T13:28:45,407 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-08T13:28:45,410 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:45,412 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T13:28:45,412 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:28:45,412 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:28:45,414 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T13:28:45,414 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:28:45,414 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:28:45,415 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731072525412Disabling compacts and flushes for region at 1731072525412Disabling writes for close at 1731072525414 (+2 ms)Writing region close event to WAL at 1731072525414Closed at 1731072525414 2024-11-08T13:28:45,417 WARN [master/6ccfea20c990:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/data/master/store/.initializing 2024-11-08T13:28:45,417 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/WALs/6ccfea20c990,40009,1731072524365 2024-11-08T13:28:45,424 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T13:28:45,438 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6ccfea20c990%2C40009%2C1731072524365, suffix=, logDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/WALs/6ccfea20c990,40009,1731072524365, archiveDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/oldWALs, maxLogs=10 2024-11-08T13:28:45,458 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/WALs/6ccfea20c990,40009,1731072524365/6ccfea20c990%2C40009%2C1731072524365.1731072525442, exclude list is [], retry=0 2024-11-08T13:28:45,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36805,DS-067499be-1a0c-4eb6-83cf-12ed63a0b51c,DISK] 2024-11-08T13:28:45,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36145,DS-d3e6e3df-bbf9-4750-9cd5-a9cb825eda4a,DISK] 2024-11-08T13:28:45,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38513,DS-4c23f8af-3e26-4caf-af50-9edfbeedfc0d,DISK] 2024-11-08T13:28:45,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-08T13:28:45,512 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/WALs/6ccfea20c990,40009,1731072524365/6ccfea20c990%2C40009%2C1731072524365.1731072525442 2024-11-08T13:28:45,513 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44267:44267),(127.0.0.1/127.0.0.1:43697:43697),(127.0.0.1/127.0.0.1:41369:41369)] 2024-11-08T13:28:45,513 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:28:45,513 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:45,516 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:28:45,517 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:28:45,551 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:28:45,572 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T13:28:45,575 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:45,577 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:28:45,578 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:28:45,581 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T13:28:45,581 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:45,582 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:45,582 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:28:45,585 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T13:28:45,585 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:45,586 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:45,586 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:28:45,588 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T13:28:45,588 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:45,589 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:45,590 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:28:45,593 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:28:45,594 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:28:45,599 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:28:45,600 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:28:45,603 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T13:28:45,606 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:28:45,611 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:45,612 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71684658, jitterRate=0.06818464398384094}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T13:28:45,622 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731072525527Initializing all the Stores at 1731072525529 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072525530 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072525530Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072525531 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072525531Cleaning up temporary data from old regions at 1731072525600 (+69 ms)Region opened successfully at 1731072525622 (+22 ms) 2024-11-08T13:28:45,623 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T13:28:45,657 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30fc7897, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6ccfea20c990/172.17.0.2:0 2024-11-08T13:28:45,684 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T13:28:45,693 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T13:28:45,693 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T13:28:45,696 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T13:28:45,697 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T13:28:45,702 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-08T13:28:45,702 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T13:28:45,730 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T13:28:45,741 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T13:28:45,743 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T13:28:45,746 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T13:28:45,748 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T13:28:45,749 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T13:28:45,752 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T13:28:45,756 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T13:28:45,757 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T13:28:45,758 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T13:28:45,759 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T13:28:45,779 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T13:28:45,780 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T13:28:45,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39565-0x1001f85d9a10001, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T13:28:45,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T13:28:45,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T13:28:45,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39565-0x1001f85d9a10001, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:45,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:45,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T13:28:45,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:45,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:45,787 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=6ccfea20c990,40009,1731072524365, sessionid=0x1001f85d9a10000, setting cluster-up flag (Was=false) 2024-11-08T13:28:45,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:45,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39565-0x1001f85d9a10001, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:45,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:45,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:45,802 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T13:28:45,804 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6ccfea20c990,40009,1731072524365 2024-11-08T13:28:45,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:45,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39565-0x1001f85d9a10001, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:45,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:45,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:45,812 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T13:28:45,814 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6ccfea20c990,40009,1731072524365 2024-11-08T13:28:45,820 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T13:28:45,880 INFO [RS:0;6ccfea20c990:39565 {}] regionserver.HRegionServer(746): ClusterId : 726d013e-8c45-4909-bccd-5b44ea40be15 2024-11-08T13:28:45,880 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(746): ClusterId : 726d013e-8c45-4909-bccd-5b44ea40be15 2024-11-08T13:28:45,881 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(746): ClusterId : 726d013e-8c45-4909-bccd-5b44ea40be15 2024-11-08T13:28:45,883 DEBUG [RS:2;6ccfea20c990:35367 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T13:28:45,883 DEBUG [RS:1;6ccfea20c990:35061 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T13:28:45,883 DEBUG [RS:0;6ccfea20c990:39565 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T13:28:45,887 DEBUG [RS:1;6ccfea20c990:35061 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T13:28:45,887 DEBUG [RS:1;6ccfea20c990:35061 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T13:28:45,887 DEBUG [RS:0;6ccfea20c990:39565 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T13:28:45,887 DEBUG [RS:0;6ccfea20c990:39565 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T13:28:45,887 DEBUG [RS:2;6ccfea20c990:35367 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T13:28:45,888 DEBUG [RS:2;6ccfea20c990:35367 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T13:28:45,887 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T13:28:45,890 DEBUG [RS:0;6ccfea20c990:39565 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T13:28:45,890 DEBUG [RS:1;6ccfea20c990:35061 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T13:28:45,890 DEBUG [RS:2;6ccfea20c990:35367 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T13:28:45,891 DEBUG [RS:0;6ccfea20c990:39565 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5215cf21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6ccfea20c990/172.17.0.2:0 2024-11-08T13:28:45,891 DEBUG [RS:2;6ccfea20c990:35367 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a3b5f22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6ccfea20c990/172.17.0.2:0 2024-11-08T13:28:45,891 DEBUG [RS:1;6ccfea20c990:35061 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fe8731e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6ccfea20c990/172.17.0.2:0 2024-11-08T13:28:45,899 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T13:28:45,908 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T13:28:45,910 DEBUG [RS:1;6ccfea20c990:35061 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;6ccfea20c990:35061 2024-11-08T13:28:45,910 DEBUG [RS:2;6ccfea20c990:35367 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;6ccfea20c990:35367 2024-11-08T13:28:45,910 DEBUG [RS:0;6ccfea20c990:39565 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;6ccfea20c990:39565 2024-11-08T13:28:45,914 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T13:28:45,914 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T13:28:45,914 INFO [RS:0;6ccfea20c990:39565 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T13:28:45,915 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T13:28:45,915 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T13:28:45,915 INFO [RS:0;6ccfea20c990:39565 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T13:28:45,915 DEBUG [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T13:28:45,915 DEBUG [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T13:28:45,915 DEBUG [RS:0;6ccfea20c990:39565 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T13:28:45,914 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 6ccfea20c990,40009,1731072524365 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T13:28:45,918 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(2659): reportForDuty to master=6ccfea20c990,40009,1731072524365 with port=35061, startcode=1731072525012 2024-11-08T13:28:45,918 INFO [RS:0;6ccfea20c990:39565 {}] regionserver.HRegionServer(2659): reportForDuty to master=6ccfea20c990,40009,1731072524365 with port=39565, startcode=1731072524927 2024-11-08T13:28:45,918 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(2659): reportForDuty to master=6ccfea20c990,40009,1731072524365 with port=35367, startcode=1731072525051 2024-11-08T13:28:45,922 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/6ccfea20c990:0, corePoolSize=5, maxPoolSize=5 2024-11-08T13:28:45,923 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/6ccfea20c990:0, corePoolSize=5, maxPoolSize=5 2024-11-08T13:28:45,923 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/6ccfea20c990:0, corePoolSize=5, maxPoolSize=5 2024-11-08T13:28:45,923 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/6ccfea20c990:0, corePoolSize=5, maxPoolSize=5 2024-11-08T13:28:45,923 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/6ccfea20c990:0, corePoolSize=10, maxPoolSize=10 2024-11-08T13:28:45,924 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:45,924 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/6ccfea20c990:0, corePoolSize=2, maxPoolSize=2 2024-11-08T13:28:45,924 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:45,926 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731072555926 2024-11-08T13:28:45,928 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T13:28:45,929 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T13:28:45,930 DEBUG [RS:0;6ccfea20c990:39565 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T13:28:45,930 DEBUG [RS:2;6ccfea20c990:35367 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T13:28:45,931 DEBUG [RS:1;6ccfea20c990:35061 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T13:28:45,932 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T13:28:45,932 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T13:28:45,933 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T13:28:45,934 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T13:28:45,934 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T13:28:45,934 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T13:28:45,935 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:45,937 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:45,938 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T13:28:45,945 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T13:28:45,947 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T13:28:45,948 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T13:28:45,953 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T13:28:45,953 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T13:28:45,958 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.large.0-1731072525955,5,FailOnTimeoutGroup] 2024-11-08T13:28:45,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741831_1007 (size=1321) 2024-11-08T13:28:45,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741831_1007 (size=1321) 2024-11-08T13:28:45,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741831_1007 (size=1321) 2024-11-08T13:28:45,971 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T13:28:45,971 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:45,973 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.small.0-1731072525958,5,FailOnTimeoutGroup] 2024-11-08T13:28:45,973 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:45,973 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T13:28:45,974 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:45,975 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:45,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741832_1008 (size=32) 2024-11-08T13:28:45,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741832_1008 (size=32) 2024-11-08T13:28:45,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741832_1008 (size=32) 2024-11-08T13:28:45,993 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:45,995 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T13:28:45,997 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T13:28:45,997 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:45,999 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:28:45,999 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T13:28:46,002 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T13:28:46,002 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:46,003 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:28:46,003 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T13:28:46,006 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T13:28:46,006 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:46,008 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:28:46,008 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T13:28:46,008 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49081, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T13:28:46,008 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49275, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T13:28:46,008 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36499, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T13:28:46,011 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T13:28:46,011 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:46,013 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:28:46,013 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T13:28:46,015 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740 2024-11-08T13:28:46,016 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740 2024-11-08T13:28:46,017 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40009 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6ccfea20c990,39565,1731072524927 2024-11-08T13:28:46,019 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T13:28:46,019 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40009 {}] master.ServerManager(517): Registering regionserver=6ccfea20c990,39565,1731072524927 2024-11-08T13:28:46,019 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T13:28:46,020 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T13:28:46,023 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T13:28:46,029 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:46,030 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74789631, jitterRate=0.11445234715938568}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T13:28:46,032 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40009 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6ccfea20c990,35367,1731072525051 2024-11-08T13:28:46,033 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40009 {}] master.ServerManager(517): Registering regionserver=6ccfea20c990,35367,1731072525051 2024-11-08T13:28:46,035 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731072525993Initializing all the Stores at 1731072525994 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072525994Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072525994Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072525994Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072525994Cleaning up temporary data from old regions at 1731072526020 (+26 ms)Region opened successfully at 1731072526035 (+15 ms) 2024-11-08T13:28:46,035 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T13:28:46,036 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T13:28:46,036 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T13:28:46,036 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T13:28:46,036 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T13:28:46,037 DEBUG [RS:0;6ccfea20c990:39565 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:46,037 DEBUG [RS:0;6ccfea20c990:39565 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43827 2024-11-08T13:28:46,037 DEBUG [RS:0;6ccfea20c990:39565 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T13:28:46,039 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40009 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6ccfea20c990,35061,1731072525012 2024-11-08T13:28:46,039 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40009 {}] master.ServerManager(517): Registering regionserver=6ccfea20c990,35061,1731072525012 2024-11-08T13:28:46,039 DEBUG [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:46,039 DEBUG [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43827 2024-11-08T13:28:46,039 DEBUG [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T13:28:46,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T13:28:46,042 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T13:28:46,042 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731072526035Disabling compacts and flushes for region at 1731072526035Disabling writes for close at 1731072526036 (+1 ms)Writing region close event to WAL at 1731072526041 (+5 ms)Closed at 1731072526041 2024-11-08T13:28:46,042 DEBUG [RS:0;6ccfea20c990:39565 {}] zookeeper.ZKUtil(111): regionserver:39565-0x1001f85d9a10001, quorum=127.0.0.1:50677, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6ccfea20c990,39565,1731072524927 2024-11-08T13:28:46,043 WARN [RS:0;6ccfea20c990:39565 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T13:28:46,043 INFO [RS:0;6ccfea20c990:39565 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T13:28:46,043 DEBUG [RS:0;6ccfea20c990:39565 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927 2024-11-08T13:28:46,045 DEBUG [RS:2;6ccfea20c990:35367 {}] zookeeper.ZKUtil(111): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6ccfea20c990,35367,1731072525051 2024-11-08T13:28:46,045 WARN [RS:2;6ccfea20c990:35367 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T13:28:46,045 INFO [RS:2;6ccfea20c990:35367 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T13:28:46,045 DEBUG [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:46,045 DEBUG [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43827 2024-11-08T13:28:46,045 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T13:28:46,045 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T13:28:46,045 DEBUG [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T13:28:46,045 DEBUG [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,35367,1731072525051 2024-11-08T13:28:46,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T13:28:46,047 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6ccfea20c990,35367,1731072525051] 2024-11-08T13:28:46,048 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6ccfea20c990,39565,1731072524927] 2024-11-08T13:28:46,048 DEBUG [RS:1;6ccfea20c990:35061 {}] zookeeper.ZKUtil(111): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6ccfea20c990,35061,1731072525012 2024-11-08T13:28:46,048 WARN [RS:1;6ccfea20c990:35061 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T13:28:46,048 INFO [RS:1;6ccfea20c990:35061 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T13:28:46,049 DEBUG [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,35061,1731072525012 2024-11-08T13:28:46,049 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6ccfea20c990,35061,1731072525012] 2024-11-08T13:28:46,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T13:28:46,068 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T13:28:46,071 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T13:28:46,079 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T13:28:46,079 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T13:28:46,079 INFO [RS:0;6ccfea20c990:39565 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T13:28:46,094 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T13:28:46,094 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T13:28:46,094 INFO [RS:0;6ccfea20c990:39565 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T13:28:46,100 INFO [RS:0;6ccfea20c990:39565 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T13:28:46,100 INFO [RS:1;6ccfea20c990:35061 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T13:28:46,100 INFO [RS:2;6ccfea20c990:35367 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T13:28:46,100 INFO [RS:1;6ccfea20c990:35061 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,100 INFO [RS:0;6ccfea20c990:39565 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,100 INFO [RS:2;6ccfea20c990:35367 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,103 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T13:28:46,103 INFO [RS:0;6ccfea20c990:39565 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T13:28:46,103 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T13:28:46,109 INFO [RS:0;6ccfea20c990:39565 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T13:28:46,109 INFO [RS:1;6ccfea20c990:35061 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T13:28:46,109 INFO [RS:2;6ccfea20c990:35367 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T13:28:46,110 INFO [RS:1;6ccfea20c990:35061 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,110 INFO [RS:2;6ccfea20c990:35367 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,110 INFO [RS:0;6ccfea20c990:39565 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,110 DEBUG [RS:2;6ccfea20c990:35367 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,110 DEBUG [RS:1;6ccfea20c990:35061 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,110 DEBUG [RS:0;6ccfea20c990:39565 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,110 DEBUG [RS:2;6ccfea20c990:35367 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,111 DEBUG [RS:0;6ccfea20c990:39565 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,111 DEBUG [RS:1;6ccfea20c990:35061 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,111 DEBUG [RS:0;6ccfea20c990:39565 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,111 DEBUG [RS:2;6ccfea20c990:35367 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,111 DEBUG [RS:1;6ccfea20c990:35061 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,111 DEBUG [RS:0;6ccfea20c990:39565 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,111 DEBUG [RS:2;6ccfea20c990:35367 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,111 DEBUG [RS:1;6ccfea20c990:35061 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,111 DEBUG [RS:0;6ccfea20c990:39565 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,111 DEBUG [RS:2;6ccfea20c990:35367 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,111 DEBUG [RS:0;6ccfea20c990:39565 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0, corePoolSize=2, maxPoolSize=2 2024-11-08T13:28:46,111 DEBUG [RS:2;6ccfea20c990:35367 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0, corePoolSize=2, maxPoolSize=2 2024-11-08T13:28:46,111 DEBUG [RS:1;6ccfea20c990:35061 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,111 DEBUG [RS:0;6ccfea20c990:39565 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,111 DEBUG [RS:2;6ccfea20c990:35367 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,111 DEBUG [RS:1;6ccfea20c990:35061 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0, corePoolSize=2, maxPoolSize=2 2024-11-08T13:28:46,111 DEBUG [RS:0;6ccfea20c990:39565 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,111 DEBUG [RS:2;6ccfea20c990:35367 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,111 DEBUG [RS:1;6ccfea20c990:35061 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,111 DEBUG [RS:0;6ccfea20c990:39565 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,111 DEBUG [RS:2;6ccfea20c990:35367 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,111 DEBUG [RS:0;6ccfea20c990:39565 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,111 DEBUG [RS:2;6ccfea20c990:35367 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,111 DEBUG [RS:1;6ccfea20c990:35061 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,111 DEBUG [RS:0;6ccfea20c990:39565 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,112 DEBUG [RS:2;6ccfea20c990:35367 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,112 DEBUG [RS:1;6ccfea20c990:35061 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,112 DEBUG [RS:0;6ccfea20c990:39565 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,112 DEBUG [RS:2;6ccfea20c990:35367 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,112 DEBUG [RS:0;6ccfea20c990:39565 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6ccfea20c990:0, corePoolSize=3, maxPoolSize=3 2024-11-08T13:28:46,112 DEBUG [RS:1;6ccfea20c990:35061 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,112 DEBUG [RS:2;6ccfea20c990:35367 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6ccfea20c990:0, corePoolSize=3, maxPoolSize=3 2024-11-08T13:28:46,112 DEBUG [RS:0;6ccfea20c990:39565 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6ccfea20c990:0, corePoolSize=3, maxPoolSize=3 2024-11-08T13:28:46,112 DEBUG [RS:1;6ccfea20c990:35061 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,112 DEBUG [RS:2;6ccfea20c990:35367 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6ccfea20c990:0, corePoolSize=3, maxPoolSize=3 2024-11-08T13:28:46,112 DEBUG [RS:1;6ccfea20c990:35061 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:28:46,112 DEBUG [RS:1;6ccfea20c990:35061 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6ccfea20c990:0, corePoolSize=3, maxPoolSize=3 2024-11-08T13:28:46,112 DEBUG [RS:1;6ccfea20c990:35061 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6ccfea20c990:0, corePoolSize=3, maxPoolSize=3 2024-11-08T13:28:46,113 INFO [RS:0;6ccfea20c990:39565 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,114 INFO [RS:0;6ccfea20c990:39565 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,114 INFO [RS:0;6ccfea20c990:39565 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,114 INFO [RS:0;6ccfea20c990:39565 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,114 INFO [RS:0;6ccfea20c990:39565 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,114 INFO [RS:0;6ccfea20c990:39565 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,39565,1731072524927-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T13:28:46,115 INFO [RS:1;6ccfea20c990:35061 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,116 INFO [RS:1;6ccfea20c990:35061 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,116 INFO [RS:1;6ccfea20c990:35061 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,116 INFO [RS:2;6ccfea20c990:35367 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,116 INFO [RS:1;6ccfea20c990:35061 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,116 INFO [RS:2;6ccfea20c990:35367 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,116 INFO [RS:1;6ccfea20c990:35061 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,116 INFO [RS:2;6ccfea20c990:35367 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,116 INFO [RS:1;6ccfea20c990:35061 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,35061,1731072525012-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T13:28:46,116 INFO [RS:2;6ccfea20c990:35367 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,116 INFO [RS:2;6ccfea20c990:35367 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,116 INFO [RS:2;6ccfea20c990:35367 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,35367,1731072525051-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T13:28:46,134 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T13:28:46,134 INFO [RS:0;6ccfea20c990:39565 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T13:28:46,136 INFO [RS:1;6ccfea20c990:35061 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,35061,1731072525012-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,136 INFO [RS:0;6ccfea20c990:39565 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,39565,1731072524927-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,137 INFO [RS:0;6ccfea20c990:39565 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,137 INFO [RS:1;6ccfea20c990:35061 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,137 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.Replication(171): 6ccfea20c990,35061,1731072525012 started 2024-11-08T13:28:46,137 INFO [RS:0;6ccfea20c990:39565 {}] regionserver.Replication(171): 6ccfea20c990,39565,1731072524927 started 2024-11-08T13:28:46,140 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T13:28:46,140 INFO [RS:2;6ccfea20c990:35367 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,35367,1731072525051-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,140 INFO [RS:2;6ccfea20c990:35367 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,141 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.Replication(171): 6ccfea20c990,35367,1731072525051 started 2024-11-08T13:28:46,154 INFO [RS:1;6ccfea20c990:35061 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,154 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(1482): Serving as 6ccfea20c990,35061,1731072525012, RpcServer on 6ccfea20c990/172.17.0.2:35061, sessionid=0x1001f85d9a10002 2024-11-08T13:28:46,155 DEBUG [RS:1;6ccfea20c990:35061 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T13:28:46,155 DEBUG [RS:1;6ccfea20c990:35061 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6ccfea20c990,35061,1731072525012 2024-11-08T13:28:46,155 DEBUG [RS:1;6ccfea20c990:35061 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ccfea20c990,35061,1731072525012' 2024-11-08T13:28:46,156 DEBUG [RS:1;6ccfea20c990:35061 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T13:28:46,157 DEBUG [RS:1;6ccfea20c990:35061 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T13:28:46,157 DEBUG [RS:1;6ccfea20c990:35061 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T13:28:46,157 DEBUG [RS:1;6ccfea20c990:35061 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T13:28:46,158 DEBUG [RS:1;6ccfea20c990:35061 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6ccfea20c990,35061,1731072525012 2024-11-08T13:28:46,158 DEBUG [RS:1;6ccfea20c990:35061 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ccfea20c990,35061,1731072525012' 2024-11-08T13:28:46,158 DEBUG [RS:1;6ccfea20c990:35061 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T13:28:46,158 DEBUG [RS:1;6ccfea20c990:35061 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T13:28:46,159 DEBUG [RS:1;6ccfea20c990:35061 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T13:28:46,159 INFO [RS:1;6ccfea20c990:35061 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T13:28:46,159 INFO [RS:1;6ccfea20c990:35061 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T13:28:46,160 INFO [RS:0;6ccfea20c990:39565 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,160 INFO [RS:0;6ccfea20c990:39565 {}] regionserver.HRegionServer(1482): Serving as 6ccfea20c990,39565,1731072524927, RpcServer on 6ccfea20c990/172.17.0.2:39565, sessionid=0x1001f85d9a10001 2024-11-08T13:28:46,160 DEBUG [RS:0;6ccfea20c990:39565 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T13:28:46,160 DEBUG [RS:0;6ccfea20c990:39565 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6ccfea20c990,39565,1731072524927 2024-11-08T13:28:46,160 DEBUG [RS:0;6ccfea20c990:39565 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ccfea20c990,39565,1731072524927' 2024-11-08T13:28:46,161 DEBUG [RS:0;6ccfea20c990:39565 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T13:28:46,161 DEBUG [RS:0;6ccfea20c990:39565 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T13:28:46,162 DEBUG [RS:0;6ccfea20c990:39565 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T13:28:46,162 DEBUG [RS:0;6ccfea20c990:39565 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T13:28:46,162 DEBUG [RS:0;6ccfea20c990:39565 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6ccfea20c990,39565,1731072524927 2024-11-08T13:28:46,162 DEBUG [RS:0;6ccfea20c990:39565 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ccfea20c990,39565,1731072524927' 2024-11-08T13:28:46,162 DEBUG [RS:0;6ccfea20c990:39565 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T13:28:46,162 INFO [RS:2;6ccfea20c990:35367 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,163 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(1482): Serving as 6ccfea20c990,35367,1731072525051, RpcServer on 6ccfea20c990/172.17.0.2:35367, sessionid=0x1001f85d9a10003 2024-11-08T13:28:46,163 DEBUG [RS:0;6ccfea20c990:39565 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T13:28:46,163 DEBUG [RS:2;6ccfea20c990:35367 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T13:28:46,163 DEBUG [RS:2;6ccfea20c990:35367 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6ccfea20c990,35367,1731072525051 2024-11-08T13:28:46,163 DEBUG [RS:2;6ccfea20c990:35367 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ccfea20c990,35367,1731072525051' 2024-11-08T13:28:46,163 DEBUG [RS:2;6ccfea20c990:35367 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T13:28:46,163 DEBUG [RS:0;6ccfea20c990:39565 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T13:28:46,163 INFO [RS:0;6ccfea20c990:39565 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T13:28:46,163 INFO [RS:0;6ccfea20c990:39565 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T13:28:46,164 DEBUG [RS:2;6ccfea20c990:35367 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T13:28:46,164 DEBUG [RS:2;6ccfea20c990:35367 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T13:28:46,164 DEBUG [RS:2;6ccfea20c990:35367 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T13:28:46,164 DEBUG [RS:2;6ccfea20c990:35367 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6ccfea20c990,35367,1731072525051 2024-11-08T13:28:46,164 DEBUG [RS:2;6ccfea20c990:35367 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ccfea20c990,35367,1731072525051' 2024-11-08T13:28:46,164 DEBUG [RS:2;6ccfea20c990:35367 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T13:28:46,165 DEBUG [RS:2;6ccfea20c990:35367 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T13:28:46,166 DEBUG [RS:2;6ccfea20c990:35367 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T13:28:46,166 INFO [RS:2;6ccfea20c990:35367 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T13:28:46,166 INFO [RS:2;6ccfea20c990:35367 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T13:28:46,222 WARN [6ccfea20c990:40009 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:28:46,264 INFO [RS:1;6ccfea20c990:35061 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T13:28:46,264 INFO [RS:0;6ccfea20c990:39565 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T13:28:46,266 INFO [RS:2;6ccfea20c990:35367 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T13:28:46,267 INFO [RS:1;6ccfea20c990:35061 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6ccfea20c990%2C35061%2C1731072525012, suffix=, logDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,35061,1731072525012, archiveDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/oldWALs, maxLogs=32 2024-11-08T13:28:46,267 INFO [RS:0;6ccfea20c990:39565 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6ccfea20c990%2C39565%2C1731072524927, suffix=, logDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927, archiveDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/oldWALs, maxLogs=32 2024-11-08T13:28:46,269 INFO [RS:2;6ccfea20c990:35367 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6ccfea20c990%2C35367%2C1731072525051, suffix=, logDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,35367,1731072525051, archiveDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/oldWALs, maxLogs=32 2024-11-08T13:28:46,287 DEBUG [RS:2;6ccfea20c990:35367 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,35367,1731072525051/6ccfea20c990%2C35367%2C1731072525051.1731072526271, exclude list is [], retry=0 2024-11-08T13:28:46,291 DEBUG [RS:1;6ccfea20c990:35061 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,35061,1731072525012/6ccfea20c990%2C35061%2C1731072525012.1731072526271, exclude list is [], retry=0 2024-11-08T13:28:46,292 DEBUG [RS:0;6ccfea20c990:39565 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927/6ccfea20c990%2C39565%2C1731072524927.1731072526271, exclude list is [], retry=0 2024-11-08T13:28:46,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38513,DS-4c23f8af-3e26-4caf-af50-9edfbeedfc0d,DISK] 2024-11-08T13:28:46,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36145,DS-d3e6e3df-bbf9-4750-9cd5-a9cb825eda4a,DISK] 2024-11-08T13:28:46,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36805,DS-067499be-1a0c-4eb6-83cf-12ed63a0b51c,DISK] 2024-11-08T13:28:46,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38513,DS-4c23f8af-3e26-4caf-af50-9edfbeedfc0d,DISK] 2024-11-08T13:28:46,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38513,DS-4c23f8af-3e26-4caf-af50-9edfbeedfc0d,DISK] 2024-11-08T13:28:46,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36805,DS-067499be-1a0c-4eb6-83cf-12ed63a0b51c,DISK] 2024-11-08T13:28:46,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36145,DS-d3e6e3df-bbf9-4750-9cd5-a9cb825eda4a,DISK] 2024-11-08T13:28:46,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36145,DS-d3e6e3df-bbf9-4750-9cd5-a9cb825eda4a,DISK] 2024-11-08T13:28:46,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36805,DS-067499be-1a0c-4eb6-83cf-12ed63a0b51c,DISK] 2024-11-08T13:28:46,299 INFO [RS:2;6ccfea20c990:35367 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,35367,1731072525051/6ccfea20c990%2C35367%2C1731072525051.1731072526271 2024-11-08T13:28:46,305 DEBUG [RS:2;6ccfea20c990:35367 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43697:43697),(127.0.0.1/127.0.0.1:41369:41369),(127.0.0.1/127.0.0.1:44267:44267)] 2024-11-08T13:28:46,309 INFO [RS:1;6ccfea20c990:35061 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,35061,1731072525012/6ccfea20c990%2C35061%2C1731072525012.1731072526271 2024-11-08T13:28:46,309 INFO [RS:0;6ccfea20c990:39565 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927/6ccfea20c990%2C39565%2C1731072524927.1731072526271 2024-11-08T13:28:46,310 DEBUG [RS:1;6ccfea20c990:35061 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43697:43697),(127.0.0.1/127.0.0.1:44267:44267),(127.0.0.1/127.0.0.1:41369:41369)] 2024-11-08T13:28:46,310 DEBUG [RS:0;6ccfea20c990:39565 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43697:43697),(127.0.0.1/127.0.0.1:41369:41369),(127.0.0.1/127.0.0.1:44267:44267)] 2024-11-08T13:28:46,477 DEBUG [6ccfea20c990:40009 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-08T13:28:46,490 DEBUG [6ccfea20c990:40009 {}] balancer.BalancerClusterState(204): Hosts are {6ccfea20c990=0} racks are {/default-rack=0} 2024-11-08T13:28:46,497 DEBUG [6ccfea20c990:40009 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T13:28:46,497 DEBUG [6ccfea20c990:40009 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T13:28:46,497 DEBUG [6ccfea20c990:40009 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-08T13:28:46,497 DEBUG [6ccfea20c990:40009 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T13:28:46,497 DEBUG [6ccfea20c990:40009 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T13:28:46,497 DEBUG [6ccfea20c990:40009 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-08T13:28:46,497 INFO [6ccfea20c990:40009 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T13:28:46,497 INFO [6ccfea20c990:40009 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T13:28:46,497 INFO [6ccfea20c990:40009 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-08T13:28:46,497 DEBUG [6ccfea20c990:40009 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T13:28:46,503 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6ccfea20c990,39565,1731072524927 2024-11-08T13:28:46,509 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ccfea20c990,39565,1731072524927, state=OPENING 2024-11-08T13:28:46,513 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T13:28:46,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:46,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39565-0x1001f85d9a10001, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:46,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:46,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:28:46,515 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:28:46,515 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:28:46,515 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:28:46,516 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:28:46,517 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T13:28:46,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6ccfea20c990,39565,1731072524927}] 2024-11-08T13:28:46,697 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T13:28:46,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39375, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T13:28:46,713 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T13:28:46,713 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T13:28:46,714 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T13:28:46,716 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6ccfea20c990%2C39565%2C1731072524927.meta, suffix=.meta, logDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927, archiveDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/oldWALs, maxLogs=32 2024-11-08T13:28:46,730 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927/6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, exclude list is [], retry=0 2024-11-08T13:28:46,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38513,DS-4c23f8af-3e26-4caf-af50-9edfbeedfc0d,DISK] 2024-11-08T13:28:46,734 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36145,DS-d3e6e3df-bbf9-4750-9cd5-a9cb825eda4a,DISK] 2024-11-08T13:28:46,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36805,DS-067499be-1a0c-4eb6-83cf-12ed63a0b51c,DISK] 2024-11-08T13:28:46,737 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927/6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta 2024-11-08T13:28:46,738 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43697:43697),(127.0.0.1/127.0.0.1:41369:41369),(127.0.0.1/127.0.0.1:44267:44267)] 2024-11-08T13:28:46,738 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:28:46,740 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T13:28:46,742 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T13:28:46,747 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T13:28:46,750 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T13:28:46,751 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:46,751 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T13:28:46,751 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T13:28:46,754 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T13:28:46,756 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T13:28:46,756 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:46,756 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:28:46,757 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T13:28:46,758 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T13:28:46,758 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:46,759 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:28:46,759 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T13:28:46,761 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T13:28:46,761 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:46,762 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:28:46,762 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T13:28:46,763 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T13:28:46,763 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:46,764 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:28:46,764 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T13:28:46,766 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740 2024-11-08T13:28:46,769 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740 2024-11-08T13:28:46,771 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T13:28:46,772 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T13:28:46,773 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T13:28:46,775 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T13:28:46,777 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61640536, jitterRate=-0.08148443698883057}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T13:28:46,777 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T13:28:46,778 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731072526752Writing region info on filesystem at 1731072526752Initializing all the Stores at 1731072526753 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072526754 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072526754Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072526754Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072526754Cleaning up temporary data from old regions at 1731072526772 (+18 ms)Running coprocessor post-open hooks at 1731072526777 (+5 ms)Region opened successfully at 1731072526778 (+1 ms) 2024-11-08T13:28:46,785 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731072526689 2024-11-08T13:28:46,794 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T13:28:46,795 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T13:28:46,796 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,39565,1731072524927 2024-11-08T13:28:46,798 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ccfea20c990,39565,1731072524927, state=OPEN 2024-11-08T13:28:46,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:28:46,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:28:46,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39565-0x1001f85d9a10001, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:28:46,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:28:46,801 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:28:46,801 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:28:46,801 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:28:46,801 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:28:46,801 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6ccfea20c990,39565,1731072524927 2024-11-08T13:28:46,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T13:28:46,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ccfea20c990,39565,1731072524927 in 282 msec 2024-11-08T13:28:46,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T13:28:46,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 755 msec 2024-11-08T13:28:46,816 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T13:28:46,816 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T13:28:46,832 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:28:46,833 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:28:46,854 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T13:28:46,856 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52071, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T13:28:46,876 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0280 sec 2024-11-08T13:28:46,877 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731072526877, completionTime=-1 2024-11-08T13:28:46,880 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-08T13:28:46,880 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T13:28:46,909 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-08T13:28:46,909 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731072586909 2024-11-08T13:28:46,909 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731072646909 2024-11-08T13:28:46,909 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-11-08T13:28:46,911 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-08T13:28:46,919 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,40009,1731072524365-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,919 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,40009,1731072524365-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,919 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,40009,1731072524365-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,920 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-6ccfea20c990:40009, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,921 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,921 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,927 DEBUG [master/6ccfea20c990:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T13:28:46,952 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.822sec 2024-11-08T13:28:46,954 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T13:28:46,955 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T13:28:46,956 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T13:28:46,957 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T13:28:46,957 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T13:28:46,958 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,40009,1731072524365-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T13:28:46,958 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,40009,1731072524365-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T13:28:46,962 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T13:28:46,963 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T13:28:46,963 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,40009,1731072524365-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:28:46,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61693573, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T13:28:46,999 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T13:28:46,999 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T13:28:47,004 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6ccfea20c990,40009,-1 for getting cluster id 2024-11-08T13:28:47,007 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T13:28:47,015 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '726d013e-8c45-4909-bccd-5b44ea40be15' 2024-11-08T13:28:47,017 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T13:28:47,017 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "726d013e-8c45-4909-bccd-5b44ea40be15" 2024-11-08T13:28:47,018 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@326ba6da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T13:28:47,018 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6ccfea20c990,40009,-1] 2024-11-08T13:28:47,020 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T13:28:47,022 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:28:47,023 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T13:28:47,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5499896f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T13:28:47,026 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:28:47,033 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:28:47,034 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T13:28:47,036 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T13:28:47,083 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=6ccfea20c990,40009,1731072524365 2024-11-08T13:28:47,085 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-08T13:28:47,091 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T13:28:47,096 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 6ccfea20c990,40009,1731072524365 2024-11-08T13:28:47,098 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6d06cd09 2024-11-08T13:28:47,099 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T13:28:47,102 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T13:28:47,114 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40009 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T13:28:47,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40009 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-08T13:28:47,128 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T13:28:47,130 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40009 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true" procId is: 4 2024-11-08T13:28:47,131 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:47,134 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T13:28:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40009 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T13:28:47,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741837_1013 (size=1020) 2024-11-08T13:28:47,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741837_1013 (size=1020) 2024-11-08T13:28:47,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741837_1013 (size=1020) 2024-11-08T13:28:47,154 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c6699738a1be7c8647568269e341753c, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,155 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 4d5935497231430a7057c66d28a9d018, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,155 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 7e4837322188444dc48d5c8632c81ce5, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741838_1014 (size=110) 2024-11-08T13:28:47,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741838_1014 (size=110) 2024-11-08T13:28:47,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741838_1014 (size=110) 2024-11-08T13:28:47,182 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,182 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 4d5935497231430a7057c66d28a9d018, disabling compactions & flushes 2024-11-08T13:28:47,182 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018. 2024-11-08T13:28:47,182 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018. 2024-11-08T13:28:47,182 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018. after waiting 0 ms 2024-11-08T13:28:47,182 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018. 2024-11-08T13:28:47,182 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018. 2024-11-08T13:28:47,183 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 4d5935497231430a7057c66d28a9d018: Waiting for close lock at 1731072527182Disabling compacts and flushes for region at 1731072527182Disabling writes for close at 1731072527182Writing region close event to WAL at 1731072527182Closed at 1731072527182 2024-11-08T13:28:47,183 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 7c8fbd70358d0bf3e3f164569432ecfe, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741839_1015 (size=107) 2024-11-08T13:28:47,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741839_1015 (size=107) 2024-11-08T13:28:47,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741839_1015 (size=107) 2024-11-08T13:28:47,187 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,187 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing c6699738a1be7c8647568269e341753c, disabling compactions & flushes 2024-11-08T13:28:47,187 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c. 2024-11-08T13:28:47,187 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c. 2024-11-08T13:28:47,188 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c. after waiting 0 ms 2024-11-08T13:28:47,188 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c. 2024-11-08T13:28:47,188 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c. 2024-11-08T13:28:47,188 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for c6699738a1be7c8647568269e341753c: Waiting for close lock at 1731072527187Disabling compacts and flushes for region at 1731072527187Disabling writes for close at 1731072527188 (+1 ms)Writing region close event to WAL at 1731072527188Closed at 1731072527188 2024-11-08T13:28:47,189 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0f0799092106200541bc0c416b0bdbb5, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741840_1016 (size=110) 2024-11-08T13:28:47,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741840_1016 (size=110) 2024-11-08T13:28:47,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741840_1016 (size=110) 2024-11-08T13:28:47,193 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,194 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 7e4837322188444dc48d5c8632c81ce5, disabling compactions & flushes 2024-11-08T13:28:47,194 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5. 2024-11-08T13:28:47,194 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5. 2024-11-08T13:28:47,194 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5. after waiting 0 ms 2024-11-08T13:28:47,194 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5. 2024-11-08T13:28:47,194 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5. 2024-11-08T13:28:47,194 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 7e4837322188444dc48d5c8632c81ce5: Waiting for close lock at 1731072527193Disabling compacts and flushes for region at 1731072527193Disabling writes for close at 1731072527194 (+1 ms)Writing region close event to WAL at 1731072527194Closed at 1731072527194 2024-11-08T13:28:47,195 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d85397130af5b1fc184790336ef5d2b8, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741841_1017 (size=110) 2024-11-08T13:28:47,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741841_1017 (size=110) 2024-11-08T13:28:47,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741841_1017 (size=110) 2024-11-08T13:28:47,206 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,207 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 7c8fbd70358d0bf3e3f164569432ecfe, disabling compactions & flushes 2024-11-08T13:28:47,207 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe. 2024-11-08T13:28:47,207 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe. 2024-11-08T13:28:47,207 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe. after waiting 0 ms 2024-11-08T13:28:47,207 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe. 2024-11-08T13:28:47,207 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe. 2024-11-08T13:28:47,207 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 7c8fbd70358d0bf3e3f164569432ecfe: Waiting for close lock at 1731072527207Disabling compacts and flushes for region at 1731072527207Disabling writes for close at 1731072527207Writing region close event to WAL at 1731072527207Closed at 1731072527207 2024-11-08T13:28:47,208 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c4e1216db90b6f396355f2b6bc308ffe, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741842_1018 (size=110) 2024-11-08T13:28:47,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741842_1018 (size=110) 2024-11-08T13:28:47,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741842_1018 (size=110) 2024-11-08T13:28:47,218 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,218 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 0f0799092106200541bc0c416b0bdbb5, disabling compactions & flushes 2024-11-08T13:28:47,218 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5. 2024-11-08T13:28:47,218 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5. 2024-11-08T13:28:47,218 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5. after waiting 0 ms 2024-11-08T13:28:47,218 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5. 2024-11-08T13:28:47,218 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5. 2024-11-08T13:28:47,218 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0f0799092106200541bc0c416b0bdbb5: Waiting for close lock at 1731072527218Disabling compacts and flushes for region at 1731072527218Disabling writes for close at 1731072527218Writing region close event to WAL at 1731072527218Closed at 1731072527218 2024-11-08T13:28:47,219 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 286367177ffe2bf304003f3b829326ac, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741843_1019 (size=110) 2024-11-08T13:28:47,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741843_1019 (size=110) 2024-11-08T13:28:47,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741843_1019 (size=110) 2024-11-08T13:28:47,228 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,228 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing d85397130af5b1fc184790336ef5d2b8, disabling compactions & flushes 2024-11-08T13:28:47,228 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8. 2024-11-08T13:28:47,228 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8. 2024-11-08T13:28:47,228 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8. after waiting 0 ms 2024-11-08T13:28:47,228 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8. 2024-11-08T13:28:47,228 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8. 2024-11-08T13:28:47,228 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for d85397130af5b1fc184790336ef5d2b8: Waiting for close lock at 1731072527228Disabling compacts and flushes for region at 1731072527228Disabling writes for close at 1731072527228Writing region close event to WAL at 1731072527228Closed at 1731072527228 2024-11-08T13:28:47,229 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 50c49cba3ff42064b0e071b4b0d3656e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741844_1020 (size=110) 2024-11-08T13:28:47,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741844_1020 (size=110) 2024-11-08T13:28:47,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741844_1020 (size=110) 2024-11-08T13:28:47,235 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,235 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing c4e1216db90b6f396355f2b6bc308ffe, disabling compactions & flushes 2024-11-08T13:28:47,235 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe. 2024-11-08T13:28:47,235 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe. 2024-11-08T13:28:47,235 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe. after waiting 0 ms 2024-11-08T13:28:47,235 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe. 2024-11-08T13:28:47,235 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe. 2024-11-08T13:28:47,236 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for c4e1216db90b6f396355f2b6bc308ffe: Waiting for close lock at 1731072527235Disabling compacts and flushes for region at 1731072527235Disabling writes for close at 1731072527235Writing region close event to WAL at 1731072527235Closed at 1731072527235 2024-11-08T13:28:47,236 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 5970d249471986ac22ae194d250f9a38, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40009 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T13:28:47,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741845_1021 (size=110) 2024-11-08T13:28:47,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741845_1021 (size=110) 2024-11-08T13:28:47,248 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,248 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 286367177ffe2bf304003f3b829326ac, disabling compactions & flushes 2024-11-08T13:28:47,248 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac. 2024-11-08T13:28:47,248 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac. 2024-11-08T13:28:47,248 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac. after waiting 0 ms 2024-11-08T13:28:47,248 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac. 2024-11-08T13:28:47,248 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac. 2024-11-08T13:28:47,248 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 286367177ffe2bf304003f3b829326ac: Waiting for close lock at 1731072527248Disabling compacts and flushes for region at 1731072527248Disabling writes for close at 1731072527248Writing region close event to WAL at 1731072527248Closed at 1731072527248 2024-11-08T13:28:47,249 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5503a38fb5ee20ab8673683ba1a1bcaa, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741845_1021 (size=110) 2024-11-08T13:28:47,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741846_1022 (size=110) 2024-11-08T13:28:47,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741846_1022 (size=110) 2024-11-08T13:28:47,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741846_1022 (size=110) 2024-11-08T13:28:47,258 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,258 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 50c49cba3ff42064b0e071b4b0d3656e, disabling compactions & flushes 2024-11-08T13:28:47,258 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e. 2024-11-08T13:28:47,259 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e. 2024-11-08T13:28:47,259 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e. after waiting 0 ms 2024-11-08T13:28:47,259 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e. 2024-11-08T13:28:47,259 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e. 2024-11-08T13:28:47,259 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 50c49cba3ff42064b0e071b4b0d3656e: Waiting for close lock at 1731072527258Disabling compacts and flushes for region at 1731072527258Disabling writes for close at 1731072527259 (+1 ms)Writing region close event to WAL at 1731072527259Closed at 1731072527259 2024-11-08T13:28:47,260 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => e178bdc5865404fdcd9e707b6918569c, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741847_1023 (size=110) 2024-11-08T13:28:47,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741847_1023 (size=110) 2024-11-08T13:28:47,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741847_1023 (size=110) 2024-11-08T13:28:47,280 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,280 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 5970d249471986ac22ae194d250f9a38, disabling compactions & flushes 2024-11-08T13:28:47,280 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38. 2024-11-08T13:28:47,280 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38. 2024-11-08T13:28:47,280 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38. after waiting 0 ms 2024-11-08T13:28:47,280 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38. 2024-11-08T13:28:47,280 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38. 2024-11-08T13:28:47,281 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 5970d249471986ac22ae194d250f9a38: Waiting for close lock at 1731072527280Disabling compacts and flushes for region at 1731072527280Disabling writes for close at 1731072527280Writing region close event to WAL at 1731072527280Closed at 1731072527280 2024-11-08T13:28:47,281 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 5ef0046eead8230ad7213c161e3532bd, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741849_1025 (size=110) 2024-11-08T13:28:47,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741849_1025 (size=110) 2024-11-08T13:28:47,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741849_1025 (size=110) 2024-11-08T13:28:47,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741848_1024 (size=110) 2024-11-08T13:28:47,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741848_1024 (size=110) 2024-11-08T13:28:47,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741848_1024 (size=110) 2024-11-08T13:28:47,287 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,287 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing e178bdc5865404fdcd9e707b6918569c, disabling compactions & flushes 2024-11-08T13:28:47,287 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c. 2024-11-08T13:28:47,287 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c. 2024-11-08T13:28:47,288 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c. after waiting 0 ms 2024-11-08T13:28:47,288 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c. 2024-11-08T13:28:47,288 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c. 2024-11-08T13:28:47,288 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for e178bdc5865404fdcd9e707b6918569c: Waiting for close lock at 1731072527287Disabling compacts and flushes for region at 1731072527287Disabling writes for close at 1731072527288 (+1 ms)Writing region close event to WAL at 1731072527288Closed at 1731072527288 2024-11-08T13:28:47,289 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 0b082e035d4bc82dc2f93642d66c000c, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,289 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,290 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 5503a38fb5ee20ab8673683ba1a1bcaa, disabling compactions & flushes 2024-11-08T13:28:47,290 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa. 2024-11-08T13:28:47,290 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa. 2024-11-08T13:28:47,290 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa. after waiting 0 ms 2024-11-08T13:28:47,290 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa. 2024-11-08T13:28:47,290 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa. 2024-11-08T13:28:47,290 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5503a38fb5ee20ab8673683ba1a1bcaa: Waiting for close lock at 1731072527290Disabling compacts and flushes for region at 1731072527290Disabling writes for close at 1731072527290Writing region close event to WAL at 1731072527290Closed at 1731072527290 2024-11-08T13:28:47,291 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5ab60b8cfddafd2fcce990e602b31aef, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741850_1026 (size=110) 2024-11-08T13:28:47,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741850_1026 (size=110) 2024-11-08T13:28:47,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741850_1026 (size=110) 2024-11-08T13:28:47,305 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,305 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 5ef0046eead8230ad7213c161e3532bd, disabling compactions & flushes 2024-11-08T13:28:47,305 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd. 2024-11-08T13:28:47,305 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd. 2024-11-08T13:28:47,305 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd. after waiting 0 ms 2024-11-08T13:28:47,305 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd. 2024-11-08T13:28:47,305 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd. 2024-11-08T13:28:47,305 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 5ef0046eead8230ad7213c161e3532bd: Waiting for close lock at 1731072527305Disabling compacts and flushes for region at 1731072527305Disabling writes for close at 1731072527305Writing region close event to WAL at 1731072527305Closed at 1731072527305 2024-11-08T13:28:47,306 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 06547784f6904a15d1d127f091988ed3, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741852_1028 (size=110) 2024-11-08T13:28:47,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741852_1028 (size=110) 2024-11-08T13:28:47,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741852_1028 (size=110) 2024-11-08T13:28:47,317 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,318 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 5ab60b8cfddafd2fcce990e602b31aef, disabling compactions & flushes 2024-11-08T13:28:47,318 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef. 2024-11-08T13:28:47,318 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef. 2024-11-08T13:28:47,318 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef. after waiting 0 ms 2024-11-08T13:28:47,318 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef. 2024-11-08T13:28:47,318 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef. 2024-11-08T13:28:47,318 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5ab60b8cfddafd2fcce990e602b31aef: Waiting for close lock at 1731072527318Disabling compacts and flushes for region at 1731072527318Disabling writes for close at 1731072527318Writing region close event to WAL at 1731072527318Closed at 1731072527318 2024-11-08T13:28:47,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741851_1027 (size=110) 2024-11-08T13:28:47,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741851_1027 (size=110) 2024-11-08T13:28:47,319 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c02aa10a67704d3f15deae347223dc13, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741851_1027 (size=110) 2024-11-08T13:28:47,324 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741853_1029 (size=110) 2024-11-08T13:28:47,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741853_1029 (size=110) 2024-11-08T13:28:47,324 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 0b082e035d4bc82dc2f93642d66c000c, disabling compactions & flushes 2024-11-08T13:28:47,324 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c. 2024-11-08T13:28:47,324 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c. 2024-11-08T13:28:47,324 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c. after waiting 0 ms 2024-11-08T13:28:47,324 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c. 2024-11-08T13:28:47,324 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c. 2024-11-08T13:28:47,325 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 0b082e035d4bc82dc2f93642d66c000c: Waiting for close lock at 1731072527324Disabling compacts and flushes for region at 1731072527324Disabling writes for close at 1731072527324Writing region close event to WAL at 1731072527324Closed at 1731072527324 2024-11-08T13:28:47,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741853_1029 (size=110) 2024-11-08T13:28:47,326 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => fcf86fec97b560cab0f86ca4727e4e57, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,326 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,326 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 06547784f6904a15d1d127f091988ed3, disabling compactions & flushes 2024-11-08T13:28:47,326 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3. 2024-11-08T13:28:47,326 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3. 2024-11-08T13:28:47,326 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3. after waiting 0 ms 2024-11-08T13:28:47,326 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3. 2024-11-08T13:28:47,326 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3. 2024-11-08T13:28:47,326 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 06547784f6904a15d1d127f091988ed3: Waiting for close lock at 1731072527326Disabling compacts and flushes for region at 1731072527326Disabling writes for close at 1731072527326Writing region close event to WAL at 1731072527326Closed at 1731072527326 2024-11-08T13:28:47,327 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 223bf549b03e7990028b7927dd897a23, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741854_1030 (size=110) 2024-11-08T13:28:47,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741854_1030 (size=110) 2024-11-08T13:28:47,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741854_1030 (size=110) 2024-11-08T13:28:47,344 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,344 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing c02aa10a67704d3f15deae347223dc13, disabling compactions & flushes 2024-11-08T13:28:47,344 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13. 2024-11-08T13:28:47,344 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13. 2024-11-08T13:28:47,344 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13. after waiting 0 ms 2024-11-08T13:28:47,344 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13. 2024-11-08T13:28:47,344 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13. 2024-11-08T13:28:47,344 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for c02aa10a67704d3f15deae347223dc13: Waiting for close lock at 1731072527344Disabling compacts and flushes for region at 1731072527344Disabling writes for close at 1731072527344Writing region close event to WAL at 1731072527344Closed at 1731072527344 2024-11-08T13:28:47,345 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 30f6256cfa21047086ecffea588e9713, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741855_1031 (size=110) 2024-11-08T13:28:47,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741855_1031 (size=110) 2024-11-08T13:28:47,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741855_1031 (size=110) 2024-11-08T13:28:47,357 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,357 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 223bf549b03e7990028b7927dd897a23, disabling compactions & flushes 2024-11-08T13:28:47,357 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23. 2024-11-08T13:28:47,357 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23. 2024-11-08T13:28:47,357 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23. after waiting 0 ms 2024-11-08T13:28:47,357 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23. 2024-11-08T13:28:47,357 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23. 2024-11-08T13:28:47,357 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 223bf549b03e7990028b7927dd897a23: Waiting for close lock at 1731072527357Disabling compacts and flushes for region at 1731072527357Disabling writes for close at 1731072527357Writing region close event to WAL at 1731072527357Closed at 1731072527357 2024-11-08T13:28:47,358 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => b7f58040b9958dcdcc78f264a8778203, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741856_1032 (size=110) 2024-11-08T13:28:47,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741856_1032 (size=110) 2024-11-08T13:28:47,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741856_1032 (size=110) 2024-11-08T13:28:47,370 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,370 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing fcf86fec97b560cab0f86ca4727e4e57, disabling compactions & flushes 2024-11-08T13:28:47,370 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57. 2024-11-08T13:28:47,370 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57. 2024-11-08T13:28:47,370 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57. after waiting 0 ms 2024-11-08T13:28:47,370 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57. 2024-11-08T13:28:47,370 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57. 2024-11-08T13:28:47,371 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for fcf86fec97b560cab0f86ca4727e4e57: Waiting for close lock at 1731072527370Disabling compacts and flushes for region at 1731072527370Disabling writes for close at 1731072527370Writing region close event to WAL at 1731072527370Closed at 1731072527370 2024-11-08T13:28:47,372 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 00f5b98ab61f3475c8968b8a7a9f6cb4, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741857_1033 (size=110) 2024-11-08T13:28:47,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741857_1033 (size=110) 2024-11-08T13:28:47,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741857_1033 (size=110) 2024-11-08T13:28:47,375 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,375 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 30f6256cfa21047086ecffea588e9713, disabling compactions & flushes 2024-11-08T13:28:47,375 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713. 2024-11-08T13:28:47,375 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713. 2024-11-08T13:28:47,375 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713. after waiting 0 ms 2024-11-08T13:28:47,375 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713. 2024-11-08T13:28:47,376 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713. 2024-11-08T13:28:47,376 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 30f6256cfa21047086ecffea588e9713: Waiting for close lock at 1731072527375Disabling compacts and flushes for region at 1731072527375Disabling writes for close at 1731072527375Writing region close event to WAL at 1731072527376 (+1 ms)Closed at 1731072527376 2024-11-08T13:28:47,376 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4fbf6f08f931761acf639331e9fd9170, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741858_1034 (size=110) 2024-11-08T13:28:47,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741858_1034 (size=110) 2024-11-08T13:28:47,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741858_1034 (size=110) 2024-11-08T13:28:47,381 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,381 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing b7f58040b9958dcdcc78f264a8778203, disabling compactions & flushes 2024-11-08T13:28:47,381 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203. 2024-11-08T13:28:47,381 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203. 2024-11-08T13:28:47,381 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203. after waiting 0 ms 2024-11-08T13:28:47,382 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203. 2024-11-08T13:28:47,382 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203. 2024-11-08T13:28:47,382 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for b7f58040b9958dcdcc78f264a8778203: Waiting for close lock at 1731072527381Disabling compacts and flushes for region at 1731072527381Disabling writes for close at 1731072527382 (+1 ms)Writing region close event to WAL at 1731072527382Closed at 1731072527382 2024-11-08T13:28:47,383 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => f973a563f81a494eeb1443f81f0bb069, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741859_1035 (size=110) 2024-11-08T13:28:47,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741859_1035 (size=110) 2024-11-08T13:28:47,391 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,392 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 00f5b98ab61f3475c8968b8a7a9f6cb4, disabling compactions & flushes 2024-11-08T13:28:47,392 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4. 2024-11-08T13:28:47,392 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4. 2024-11-08T13:28:47,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741859_1035 (size=110) 2024-11-08T13:28:47,392 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4. after waiting 0 ms 2024-11-08T13:28:47,392 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4. 2024-11-08T13:28:47,392 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4. 2024-11-08T13:28:47,392 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 00f5b98ab61f3475c8968b8a7a9f6cb4: Waiting for close lock at 1731072527392Disabling compacts and flushes for region at 1731072527392Disabling writes for close at 1731072527392Writing region close event to WAL at 1731072527392Closed at 1731072527392 2024-11-08T13:28:47,393 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 99abbd965e76aa0c5fe5a1011e45a9ae, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741860_1036 (size=110) 2024-11-08T13:28:47,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741860_1036 (size=110) 2024-11-08T13:28:47,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741860_1036 (size=110) 2024-11-08T13:28:47,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741861_1037 (size=110) 2024-11-08T13:28:47,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741861_1037 (size=110) 2024-11-08T13:28:47,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741861_1037 (size=110) 2024-11-08T13:28:47,402 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,402 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 4fbf6f08f931761acf639331e9fd9170, disabling compactions & flushes 2024-11-08T13:28:47,402 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170. 2024-11-08T13:28:47,402 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170. 2024-11-08T13:28:47,402 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170. after waiting 0 ms 2024-11-08T13:28:47,402 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170. 2024-11-08T13:28:47,402 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170. 2024-11-08T13:28:47,402 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4fbf6f08f931761acf639331e9fd9170: Waiting for close lock at 1731072527402Disabling compacts and flushes for region at 1731072527402Disabling writes for close at 1731072527402Writing region close event to WAL at 1731072527402Closed at 1731072527402 2024-11-08T13:28:47,402 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,402 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing f973a563f81a494eeb1443f81f0bb069, disabling compactions & flushes 2024-11-08T13:28:47,403 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069. 2024-11-08T13:28:47,403 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069. 2024-11-08T13:28:47,403 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e1c2472f75aa6a0ed9ba5017889bd71b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f 2024-11-08T13:28:47,403 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069. after waiting 0 ms 2024-11-08T13:28:47,403 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069. 2024-11-08T13:28:47,403 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069. 2024-11-08T13:28:47,403 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for f973a563f81a494eeb1443f81f0bb069: Waiting for close lock at 1731072527402Disabling compacts and flushes for region at 1731072527402Disabling writes for close at 1731072527403 (+1 ms)Writing region close event to WAL at 1731072527403Closed at 1731072527403 2024-11-08T13:28:47,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741862_1038 (size=110) 2024-11-08T13:28:47,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741862_1038 (size=110) 2024-11-08T13:28:47,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741862_1038 (size=110) 2024-11-08T13:28:47,410 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,410 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 99abbd965e76aa0c5fe5a1011e45a9ae, disabling compactions & flushes 2024-11-08T13:28:47,410 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae. 2024-11-08T13:28:47,410 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae. 2024-11-08T13:28:47,411 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae. after waiting 0 ms 2024-11-08T13:28:47,411 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae. 2024-11-08T13:28:47,411 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae. 2024-11-08T13:28:47,411 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 99abbd965e76aa0c5fe5a1011e45a9ae: Waiting for close lock at 1731072527410Disabling compacts and flushes for region at 1731072527410Disabling writes for close at 1731072527411 (+1 ms)Writing region close event to WAL at 1731072527411Closed at 1731072527411 2024-11-08T13:28:47,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741863_1039 (size=107) 2024-11-08T13:28:47,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741863_1039 (size=107) 2024-11-08T13:28:47,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741863_1039 (size=107) 2024-11-08T13:28:47,418 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,418 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing e1c2472f75aa6a0ed9ba5017889bd71b, disabling compactions & flushes 2024-11-08T13:28:47,418 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b. 2024-11-08T13:28:47,418 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b. 2024-11-08T13:28:47,418 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b. after waiting 0 ms 2024-11-08T13:28:47,418 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b. 2024-11-08T13:28:47,418 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b. 2024-11-08T13:28:47,418 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for e1c2472f75aa6a0ed9ba5017889bd71b: Waiting for close lock at 1731072527418Disabling compacts and flushes for region at 1731072527418Disabling writes for close at 1731072527418Writing region close event to WAL at 1731072527418Closed at 1731072527418 2024-11-08T13:28:47,421 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T13:28:47,426 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,426 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c.","families":{"info":[{"qualifier":"regioninfo","vlen":106,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,426 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,427 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,427 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,427 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,427 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,427 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,428 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,428 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,428 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,428 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,428 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,428 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,429 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,429 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,429 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,429 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,429 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,429 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,429 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,430 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,430 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,430 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,430 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,430 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b.","families":{"info":[{"qualifier":"regioninfo","vlen":106,"tag":[],"timestamp":"1731072527421"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072527421"}]},"ts":"1731072527421"} 2024-11-08T13:28:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40009 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T13:28:47,477 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-08T13:28:47,479 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T13:28:47,481 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731072527479"}]},"ts":"1731072527479"} 2024-11-08T13:28:47,486 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, state=ENABLING in hbase:meta 2024-11-08T13:28:47,487 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {6ccfea20c990=0} racks are {/default-rack=0} 2024-11-08T13:28:47,529 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T13:28:47,529 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T13:28:47,529 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-08T13:28:47,530 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T13:28:47,530 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T13:28:47,530 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-08T13:28:47,530 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T13:28:47,530 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T13:28:47,530 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-08T13:28:47,530 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T13:28:47,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c6699738a1be7c8647568269e341753c, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4d5935497231430a7057c66d28a9d018, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7e4837322188444dc48d5c8632c81ce5, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7c8fbd70358d0bf3e3f164569432ecfe, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0f0799092106200541bc0c416b0bdbb5, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d85397130af5b1fc184790336ef5d2b8, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c4e1216db90b6f396355f2b6bc308ffe, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=286367177ffe2bf304003f3b829326ac, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=50c49cba3ff42064b0e071b4b0d3656e, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5970d249471986ac22ae194d250f9a38, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5503a38fb5ee20ab8673683ba1a1bcaa, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e178bdc5865404fdcd9e707b6918569c, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ef0046eead8230ad7213c161e3532bd, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0b082e035d4bc82dc2f93642d66c000c, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ab60b8cfddafd2fcce990e602b31aef, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06547784f6904a15d1d127f091988ed3, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c02aa10a67704d3f15deae347223dc13, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fcf86fec97b560cab0f86ca4727e4e57, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=223bf549b03e7990028b7927dd897a23, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=30f6256cfa21047086ecffea588e9713, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b7f58040b9958dcdcc78f264a8778203, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=00f5b98ab61f3475c8968b8a7a9f6cb4, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4fbf6f08f931761acf639331e9fd9170, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f973a563f81a494eeb1443f81f0bb069, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=99abbd965e76aa0c5fe5a1011e45a9ae, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e1c2472f75aa6a0ed9ba5017889bd71b, ASSIGN}] 2024-11-08T13:28:47,542 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e1c2472f75aa6a0ed9ba5017889bd71b, ASSIGN 2024-11-08T13:28:47,543 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e1c2472f75aa6a0ed9ba5017889bd71b, ASSIGN; state=OFFLINE, location=6ccfea20c990,35061,1731072525012; forceNewPlan=false, retain=false 2024-11-08T13:28:47,544 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=99abbd965e76aa0c5fe5a1011e45a9ae, ASSIGN 2024-11-08T13:28:47,545 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=99abbd965e76aa0c5fe5a1011e45a9ae, ASSIGN; state=OFFLINE, location=6ccfea20c990,39565,1731072524927; forceNewPlan=false, retain=false 2024-11-08T13:28:47,547 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f973a563f81a494eeb1443f81f0bb069, ASSIGN 2024-11-08T13:28:47,548 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f973a563f81a494eeb1443f81f0bb069, ASSIGN; state=OFFLINE, location=6ccfea20c990,35367,1731072525051; forceNewPlan=false, retain=false 2024-11-08T13:28:47,549 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4fbf6f08f931761acf639331e9fd9170, ASSIGN 2024-11-08T13:28:47,551 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4fbf6f08f931761acf639331e9fd9170, ASSIGN; state=OFFLINE, location=6ccfea20c990,35061,1731072525012; forceNewPlan=false, retain=false 2024-11-08T13:28:47,552 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=00f5b98ab61f3475c8968b8a7a9f6cb4, ASSIGN 2024-11-08T13:28:47,553 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=00f5b98ab61f3475c8968b8a7a9f6cb4, ASSIGN; state=OFFLINE, location=6ccfea20c990,39565,1731072524927; forceNewPlan=false, retain=false 2024-11-08T13:28:47,555 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b7f58040b9958dcdcc78f264a8778203, ASSIGN 2024-11-08T13:28:47,556 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b7f58040b9958dcdcc78f264a8778203, ASSIGN; state=OFFLINE, location=6ccfea20c990,35367,1731072525051; forceNewPlan=false, retain=false 2024-11-08T13:28:47,557 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=30f6256cfa21047086ecffea588e9713, ASSIGN 2024-11-08T13:28:47,558 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=30f6256cfa21047086ecffea588e9713, ASSIGN; state=OFFLINE, location=6ccfea20c990,35061,1731072525012; forceNewPlan=false, retain=false 2024-11-08T13:28:47,559 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=223bf549b03e7990028b7927dd897a23, ASSIGN 2024-11-08T13:28:47,560 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=223bf549b03e7990028b7927dd897a23, ASSIGN; state=OFFLINE, location=6ccfea20c990,35367,1731072525051; forceNewPlan=false, retain=false 2024-11-08T13:28:47,561 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fcf86fec97b560cab0f86ca4727e4e57, ASSIGN 2024-11-08T13:28:47,563 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fcf86fec97b560cab0f86ca4727e4e57, ASSIGN; state=OFFLINE, location=6ccfea20c990,39565,1731072524927; forceNewPlan=false, retain=false 2024-11-08T13:28:47,564 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c02aa10a67704d3f15deae347223dc13, ASSIGN 2024-11-08T13:28:47,566 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c02aa10a67704d3f15deae347223dc13, ASSIGN; state=OFFLINE, location=6ccfea20c990,35061,1731072525012; forceNewPlan=false, retain=false 2024-11-08T13:28:47,567 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06547784f6904a15d1d127f091988ed3, ASSIGN 2024-11-08T13:28:47,569 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06547784f6904a15d1d127f091988ed3, ASSIGN; state=OFFLINE, location=6ccfea20c990,39565,1731072524927; forceNewPlan=false, retain=false 2024-11-08T13:28:47,570 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ab60b8cfddafd2fcce990e602b31aef, ASSIGN 2024-11-08T13:28:47,571 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ab60b8cfddafd2fcce990e602b31aef, ASSIGN; state=OFFLINE, location=6ccfea20c990,35061,1731072525012; forceNewPlan=false, retain=false 2024-11-08T13:28:47,573 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0b082e035d4bc82dc2f93642d66c000c, ASSIGN 2024-11-08T13:28:47,574 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0b082e035d4bc82dc2f93642d66c000c, ASSIGN; state=OFFLINE, location=6ccfea20c990,35367,1731072525051; forceNewPlan=false, retain=false 2024-11-08T13:28:47,576 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ef0046eead8230ad7213c161e3532bd, ASSIGN 2024-11-08T13:28:47,578 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ef0046eead8230ad7213c161e3532bd, ASSIGN; state=OFFLINE, location=6ccfea20c990,39565,1731072524927; forceNewPlan=false, retain=false 2024-11-08T13:28:47,580 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e178bdc5865404fdcd9e707b6918569c, ASSIGN 2024-11-08T13:28:47,581 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e178bdc5865404fdcd9e707b6918569c, ASSIGN; state=OFFLINE, location=6ccfea20c990,35061,1731072525012; forceNewPlan=false, retain=false 2024-11-08T13:28:47,583 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5503a38fb5ee20ab8673683ba1a1bcaa, ASSIGN 2024-11-08T13:28:47,585 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5503a38fb5ee20ab8673683ba1a1bcaa, ASSIGN; state=OFFLINE, location=6ccfea20c990,35367,1731072525051; forceNewPlan=false, retain=false 2024-11-08T13:28:47,586 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5970d249471986ac22ae194d250f9a38, ASSIGN 2024-11-08T13:28:47,588 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5970d249471986ac22ae194d250f9a38, ASSIGN; state=OFFLINE, location=6ccfea20c990,39565,1731072524927; forceNewPlan=false, retain=false 2024-11-08T13:28:47,589 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=50c49cba3ff42064b0e071b4b0d3656e, ASSIGN 2024-11-08T13:28:47,591 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=50c49cba3ff42064b0e071b4b0d3656e, ASSIGN; state=OFFLINE, location=6ccfea20c990,35367,1731072525051; forceNewPlan=false, retain=false 2024-11-08T13:28:47,592 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=286367177ffe2bf304003f3b829326ac, ASSIGN 2024-11-08T13:28:47,594 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=286367177ffe2bf304003f3b829326ac, ASSIGN; state=OFFLINE, location=6ccfea20c990,35061,1731072525012; forceNewPlan=false, retain=false 2024-11-08T13:28:47,596 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c4e1216db90b6f396355f2b6bc308ffe, ASSIGN 2024-11-08T13:28:47,598 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c4e1216db90b6f396355f2b6bc308ffe, ASSIGN; state=OFFLINE, location=6ccfea20c990,39565,1731072524927; forceNewPlan=false, retain=false 2024-11-08T13:28:47,600 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d85397130af5b1fc184790336ef5d2b8, ASSIGN 2024-11-08T13:28:47,602 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d85397130af5b1fc184790336ef5d2b8, ASSIGN; state=OFFLINE, location=6ccfea20c990,35367,1731072525051; forceNewPlan=false, retain=false 2024-11-08T13:28:47,604 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0f0799092106200541bc0c416b0bdbb5, ASSIGN 2024-11-08T13:28:47,606 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0f0799092106200541bc0c416b0bdbb5, ASSIGN; state=OFFLINE, location=6ccfea20c990,35061,1731072525012; forceNewPlan=false, retain=false 2024-11-08T13:28:47,607 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7c8fbd70358d0bf3e3f164569432ecfe, ASSIGN 2024-11-08T13:28:47,609 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7c8fbd70358d0bf3e3f164569432ecfe, ASSIGN; state=OFFLINE, location=6ccfea20c990,39565,1731072524927; forceNewPlan=false, retain=false 2024-11-08T13:28:47,611 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7e4837322188444dc48d5c8632c81ce5, ASSIGN 2024-11-08T13:28:47,613 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7e4837322188444dc48d5c8632c81ce5, ASSIGN; state=OFFLINE, location=6ccfea20c990,35367,1731072525051; forceNewPlan=false, retain=false 2024-11-08T13:28:47,615 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4d5935497231430a7057c66d28a9d018, ASSIGN 2024-11-08T13:28:47,617 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4d5935497231430a7057c66d28a9d018, ASSIGN; state=OFFLINE, location=6ccfea20c990,39565,1731072524927; forceNewPlan=false, retain=false 2024-11-08T13:28:47,618 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c6699738a1be7c8647568269e341753c, ASSIGN 2024-11-08T13:28:47,620 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c6699738a1be7c8647568269e341753c, ASSIGN; state=OFFLINE, location=6ccfea20c990,35061,1731072525012; forceNewPlan=false, retain=false 2024-11-08T13:28:47,699 INFO [6ccfea20c990:40009 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-08T13:28:47,701 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=99abbd965e76aa0c5fe5a1011e45a9ae, regionState=OPENING, regionLocation=6ccfea20c990,39565,1731072524927 2024-11-08T13:28:47,702 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=00f5b98ab61f3475c8968b8a7a9f6cb4, regionState=OPENING, regionLocation=6ccfea20c990,39565,1731072524927 2024-11-08T13:28:47,703 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=fcf86fec97b560cab0f86ca4727e4e57, regionState=OPENING, regionLocation=6ccfea20c990,39565,1731072524927 2024-11-08T13:28:47,705 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=06547784f6904a15d1d127f091988ed3, regionState=OPENING, regionLocation=6ccfea20c990,39565,1731072524927 2024-11-08T13:28:47,705 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=99abbd965e76aa0c5fe5a1011e45a9ae, ASSIGN because future has completed 2024-11-08T13:28:47,706 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=5ef0046eead8230ad7213c161e3532bd, regionState=OPENING, regionLocation=6ccfea20c990,39565,1731072524927 2024-11-08T13:28:47,707 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=5970d249471986ac22ae194d250f9a38, regionState=OPENING, regionLocation=6ccfea20c990,39565,1731072524927 2024-11-08T13:28:47,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99abbd965e76aa0c5fe5a1011e45a9ae, server=6ccfea20c990,39565,1731072524927}] 2024-11-08T13:28:47,709 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=00f5b98ab61f3475c8968b8a7a9f6cb4, ASSIGN because future has completed 2024-11-08T13:28:47,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fcf86fec97b560cab0f86ca4727e4e57, ASSIGN because future has completed 2024-11-08T13:28:47,714 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06547784f6904a15d1d127f091988ed3, ASSIGN because future has completed 2024-11-08T13:28:47,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ef0046eead8230ad7213c161e3532bd, ASSIGN because future has completed 2024-11-08T13:28:47,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06547784f6904a15d1d127f091988ed3, server=6ccfea20c990,39565,1731072524927}] 2024-11-08T13:28:47,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5970d249471986ac22ae194d250f9a38, ASSIGN because future has completed 2024-11-08T13:28:47,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5970d249471986ac22ae194d250f9a38, server=6ccfea20c990,39565,1731072524927}] 2024-11-08T13:28:47,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ef0046eead8230ad7213c161e3532bd, server=6ccfea20c990,39565,1731072524927}] 2024-11-08T13:28:47,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure fcf86fec97b560cab0f86ca4727e4e57, server=6ccfea20c990,39565,1731072524927}] 2024-11-08T13:28:47,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00f5b98ab61f3475c8968b8a7a9f6cb4, server=6ccfea20c990,39565,1731072524927}] 2024-11-08T13:28:47,755 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=c4e1216db90b6f396355f2b6bc308ffe, regionState=OPENING, regionLocation=6ccfea20c990,39565,1731072524927 2024-11-08T13:28:47,756 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=7c8fbd70358d0bf3e3f164569432ecfe, regionState=OPENING, regionLocation=6ccfea20c990,39565,1731072524927 2024-11-08T13:28:47,757 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=4d5935497231430a7057c66d28a9d018, regionState=OPENING, regionLocation=6ccfea20c990,39565,1731072524927 2024-11-08T13:28:47,758 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=f973a563f81a494eeb1443f81f0bb069, regionState=OPENING, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:28:47,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c4e1216db90b6f396355f2b6bc308ffe, ASSIGN because future has completed 2024-11-08T13:28:47,759 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=b7f58040b9958dcdcc78f264a8778203, regionState=OPENING, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:28:47,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4e1216db90b6f396355f2b6bc308ffe, server=6ccfea20c990,39565,1731072524927}] 2024-11-08T13:28:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40009 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T13:28:47,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7c8fbd70358d0bf3e3f164569432ecfe, ASSIGN because future has completed 2024-11-08T13:28:47,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4d5935497231430a7057c66d28a9d018, ASSIGN because future has completed 2024-11-08T13:28:47,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f973a563f81a494eeb1443f81f0bb069, ASSIGN because future has completed 2024-11-08T13:28:47,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b7f58040b9958dcdcc78f264a8778203, ASSIGN because future has completed 2024-11-08T13:28:47,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7f58040b9958dcdcc78f264a8778203, server=6ccfea20c990,35367,1731072525051}] 2024-11-08T13:28:47,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure f973a563f81a494eeb1443f81f0bb069, server=6ccfea20c990,35367,1731072525051}] 2024-11-08T13:28:47,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d5935497231430a7057c66d28a9d018, server=6ccfea20c990,39565,1731072524927}] 2024-11-08T13:28:47,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c8fbd70358d0bf3e3f164569432ecfe, server=6ccfea20c990,39565,1731072524927}] 2024-11-08T13:28:47,793 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=223bf549b03e7990028b7927dd897a23, regionState=OPENING, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:28:47,794 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=0b082e035d4bc82dc2f93642d66c000c, regionState=OPENING, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:28:47,795 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=5503a38fb5ee20ab8673683ba1a1bcaa, regionState=OPENING, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:28:47,796 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=50c49cba3ff42064b0e071b4b0d3656e, regionState=OPENING, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:28:47,797 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=d85397130af5b1fc184790336ef5d2b8, regionState=OPENING, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:28:47,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=223bf549b03e7990028b7927dd897a23, ASSIGN because future has completed 2024-11-08T13:28:47,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 223bf549b03e7990028b7927dd897a23, server=6ccfea20c990,35367,1731072525051}] 2024-11-08T13:28:47,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0b082e035d4bc82dc2f93642d66c000c, ASSIGN because future has completed 2024-11-08T13:28:47,802 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5503a38fb5ee20ab8673683ba1a1bcaa, ASSIGN because future has completed 2024-11-08T13:28:47,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=50c49cba3ff42064b0e071b4b0d3656e, ASSIGN because future has completed 2024-11-08T13:28:47,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d85397130af5b1fc184790336ef5d2b8, ASSIGN because future has completed 2024-11-08T13:28:47,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure d85397130af5b1fc184790336ef5d2b8, server=6ccfea20c990,35367,1731072525051}] 2024-11-08T13:28:47,812 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 50c49cba3ff42064b0e071b4b0d3656e, server=6ccfea20c990,35367,1731072525051}] 2024-11-08T13:28:47,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5503a38fb5ee20ab8673683ba1a1bcaa, server=6ccfea20c990,35367,1731072525051}] 2024-11-08T13:28:47,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b082e035d4bc82dc2f93642d66c000c, server=6ccfea20c990,35367,1731072525051}] 2024-11-08T13:28:47,826 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7e4837322188444dc48d5c8632c81ce5, regionState=OPENING, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:28:47,827 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=e1c2472f75aa6a0ed9ba5017889bd71b, regionState=OPENING, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:28:47,827 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=4fbf6f08f931761acf639331e9fd9170, regionState=OPENING, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:28:47,828 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=30f6256cfa21047086ecffea588e9713, regionState=OPENING, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:28:47,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7e4837322188444dc48d5c8632c81ce5, ASSIGN because future has completed 2024-11-08T13:28:47,829 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=c02aa10a67704d3f15deae347223dc13, regionState=OPENING, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:28:47,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7e4837322188444dc48d5c8632c81ce5, server=6ccfea20c990,35367,1731072525051}] 2024-11-08T13:28:47,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e1c2472f75aa6a0ed9ba5017889bd71b, ASSIGN because future has completed 2024-11-08T13:28:47,833 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4fbf6f08f931761acf639331e9fd9170, ASSIGN because future has completed 2024-11-08T13:28:47,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=30f6256cfa21047086ecffea588e9713, ASSIGN because future has completed 2024-11-08T13:28:47,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30f6256cfa21047086ecffea588e9713, server=6ccfea20c990,35061,1731072525012}] 2024-11-08T13:28:47,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c02aa10a67704d3f15deae347223dc13, ASSIGN because future has completed 2024-11-08T13:28:47,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure c02aa10a67704d3f15deae347223dc13, server=6ccfea20c990,35061,1731072525012}] 2024-11-08T13:28:47,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4fbf6f08f931761acf639331e9fd9170, server=6ccfea20c990,35061,1731072525012}] 2024-11-08T13:28:47,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure e1c2472f75aa6a0ed9ba5017889bd71b, server=6ccfea20c990,35061,1731072525012}] 2024-11-08T13:28:47,853 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=5ab60b8cfddafd2fcce990e602b31aef, regionState=OPENING, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:28:47,854 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=e178bdc5865404fdcd9e707b6918569c, regionState=OPENING, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:28:47,854 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=286367177ffe2bf304003f3b829326ac, regionState=OPENING, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:28:47,855 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=0f0799092106200541bc0c416b0bdbb5, regionState=OPENING, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:28:47,856 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c6699738a1be7c8647568269e341753c, regionState=OPENING, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:28:47,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ab60b8cfddafd2fcce990e602b31aef, ASSIGN because future has completed 2024-11-08T13:28:47,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ab60b8cfddafd2fcce990e602b31aef, server=6ccfea20c990,35061,1731072525012}] 2024-11-08T13:28:47,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e178bdc5865404fdcd9e707b6918569c, ASSIGN because future has completed 2024-11-08T13:28:47,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=286367177ffe2bf304003f3b829326ac, ASSIGN because future has completed 2024-11-08T13:28:47,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0f0799092106200541bc0c416b0bdbb5, ASSIGN because future has completed 2024-11-08T13:28:47,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c6699738a1be7c8647568269e341753c, ASSIGN because future has completed 2024-11-08T13:28:47,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6699738a1be7c8647568269e341753c, server=6ccfea20c990,35061,1731072525012}] 2024-11-08T13:28:47,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0f0799092106200541bc0c416b0bdbb5, server=6ccfea20c990,35061,1731072525012}] 2024-11-08T13:28:47,870 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38. 2024-11-08T13:28:47,870 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 5970d249471986ac22ae194d250f9a38, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-08T13:28:47,871 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 5970d249471986ac22ae194d250f9a38 2024-11-08T13:28:47,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 286367177ffe2bf304003f3b829326ac, server=6ccfea20c990,35061,1731072525012}] 2024-11-08T13:28:47,871 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,871 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 5970d249471986ac22ae194d250f9a38 2024-11-08T13:28:47,871 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 5970d249471986ac22ae194d250f9a38 2024-11-08T13:28:47,874 INFO [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 5970d249471986ac22ae194d250f9a38 2024-11-08T13:28:47,876 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure e178bdc5865404fdcd9e707b6918569c, server=6ccfea20c990,35061,1731072525012}] 2024-11-08T13:28:47,876 INFO [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5970d249471986ac22ae194d250f9a38 columnFamilyName colfamily11 2024-11-08T13:28:47,876 DEBUG [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:47,877 INFO [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] regionserver.HStore(327): Store=5970d249471986ac22ae194d250f9a38/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:47,878 INFO [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 5970d249471986ac22ae194d250f9a38 2024-11-08T13:28:47,880 INFO [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5970d249471986ac22ae194d250f9a38 columnFamilyName colfamily21 2024-11-08T13:28:47,880 DEBUG [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:47,881 INFO [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] regionserver.HStore(327): Store=5970d249471986ac22ae194d250f9a38/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:47,881 INFO [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 5970d249471986ac22ae194d250f9a38 2024-11-08T13:28:47,883 INFO [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5970d249471986ac22ae194d250f9a38 columnFamilyName colfamily31 2024-11-08T13:28:47,883 DEBUG [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:47,884 INFO [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] regionserver.HStore(327): Store=5970d249471986ac22ae194d250f9a38/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:47,884 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 5970d249471986ac22ae194d250f9a38 2024-11-08T13:28:47,885 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38 2024-11-08T13:28:47,885 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38 2024-11-08T13:28:47,887 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 5970d249471986ac22ae194d250f9a38 2024-11-08T13:28:47,887 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 5970d249471986ac22ae194d250f9a38 2024-11-08T13:28:47,887 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:47,889 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 5970d249471986ac22ae194d250f9a38 2024-11-08T13:28:47,892 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:47,893 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 5970d249471986ac22ae194d250f9a38; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72387589, jitterRate=0.07865913212299347}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:47,893 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5970d249471986ac22ae194d250f9a38 2024-11-08T13:28:47,894 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 5970d249471986ac22ae194d250f9a38: Running coprocessor pre-open hook at 1731072527872Writing region info on filesystem at 1731072527872Initializing all the Stores at 1731072527873 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072527873Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072527873Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072527873Cleaning up temporary data from old regions at 1731072527887 (+14 ms)Running coprocessor post-open hooks at 1731072527893 (+6 ms)Region opened successfully at 1731072527894 (+1 ms) 2024-11-08T13:28:47,896 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38., pid=33, masterSystemTime=1731072527863 2024-11-08T13:28:47,899 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38. 2024-11-08T13:28:47,899 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38. 2024-11-08T13:28:47,899 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018. 2024-11-08T13:28:47,899 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 4d5935497231430a7057c66d28a9d018, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-08T13:28:47,899 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=5970d249471986ac22ae194d250f9a38, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,39565,1731072524927 2024-11-08T13:28:47,900 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 4d5935497231430a7057c66d28a9d018 2024-11-08T13:28:47,900 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,900 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 4d5935497231430a7057c66d28a9d018 2024-11-08T13:28:47,900 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 4d5935497231430a7057c66d28a9d018 2024-11-08T13:28:47,903 INFO [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 4d5935497231430a7057c66d28a9d018 2024-11-08T13:28:47,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5970d249471986ac22ae194d250f9a38, server=6ccfea20c990,39565,1731072524927 because future has completed 2024-11-08T13:28:47,907 INFO [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d5935497231430a7057c66d28a9d018 columnFamilyName colfamily11 2024-11-08T13:28:47,907 DEBUG [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:47,908 INFO [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] regionserver.HStore(327): Store=4d5935497231430a7057c66d28a9d018/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:47,908 INFO [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 4d5935497231430a7057c66d28a9d018 2024-11-08T13:28:47,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=14 2024-11-08T13:28:47,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 5970d249471986ac22ae194d250f9a38, server=6ccfea20c990,39565,1731072524927 in 174 msec 2024-11-08T13:28:47,911 INFO [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d5935497231430a7057c66d28a9d018 columnFamilyName colfamily21 2024-11-08T13:28:47,911 DEBUG [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:47,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5970d249471986ac22ae194d250f9a38, ASSIGN in 377 msec 2024-11-08T13:28:47,912 INFO [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] regionserver.HStore(327): Store=4d5935497231430a7057c66d28a9d018/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:47,912 INFO [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 4d5935497231430a7057c66d28a9d018 2024-11-08T13:28:47,914 INFO [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d5935497231430a7057c66d28a9d018 columnFamilyName colfamily31 2024-11-08T13:28:47,914 DEBUG [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:47,915 INFO [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] regionserver.HStore(327): Store=4d5935497231430a7057c66d28a9d018/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:47,915 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 4d5935497231430a7057c66d28a9d018 2024-11-08T13:28:47,916 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018 2024-11-08T13:28:47,917 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018 2024-11-08T13:28:47,918 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 4d5935497231430a7057c66d28a9d018 2024-11-08T13:28:47,918 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 4d5935497231430a7057c66d28a9d018 2024-11-08T13:28:47,919 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:47,921 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 4d5935497231430a7057c66d28a9d018 2024-11-08T13:28:47,922 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T13:28:47,924 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:47,925 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 4d5935497231430a7057c66d28a9d018; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64462519, jitterRate=-0.039433613419532776}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:47,925 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4d5935497231430a7057c66d28a9d018 2024-11-08T13:28:47,925 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 4d5935497231430a7057c66d28a9d018: Running coprocessor pre-open hook at 1731072527900Writing region info on filesystem at 1731072527900Initializing all the Stores at 1731072527902 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072527902Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072527902Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072527902Cleaning up temporary data from old regions at 1731072527918 (+16 ms)Running coprocessor post-open hooks at 1731072527925 (+7 ms)Region opened successfully at 1731072527925 2024-11-08T13:28:47,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46943, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T13:28:47,927 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018., pid=40, masterSystemTime=1731072527863 2024-11-08T13:28:47,930 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018. 2024-11-08T13:28:47,930 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018. 2024-11-08T13:28:47,930 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae. 2024-11-08T13:28:47,931 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 99abbd965e76aa0c5fe5a1011e45a9ae, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-08T13:28:47,931 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=4d5935497231430a7057c66d28a9d018, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,39565,1731072524927 2024-11-08T13:28:47,931 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:28:47,931 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,932 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:28:47,932 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:28:47,932 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23. 2024-11-08T13:28:47,932 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 223bf549b03e7990028b7927dd897a23, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-08T13:28:47,933 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 223bf549b03e7990028b7927dd897a23 2024-11-08T13:28:47,933 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,933 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 223bf549b03e7990028b7927dd897a23 2024-11-08T13:28:47,933 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 223bf549b03e7990028b7927dd897a23 2024-11-08T13:28:47,935 INFO [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:28:47,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d5935497231430a7057c66d28a9d018, server=6ccfea20c990,39565,1731072524927 because future has completed 2024-11-08T13:28:47,935 INFO [StoreOpener-223bf549b03e7990028b7927dd897a23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 223bf549b03e7990028b7927dd897a23 2024-11-08T13:28:47,937 INFO [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99abbd965e76aa0c5fe5a1011e45a9ae columnFamilyName colfamily11 2024-11-08T13:28:47,937 INFO [StoreOpener-223bf549b03e7990028b7927dd897a23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 223bf549b03e7990028b7927dd897a23 columnFamilyName colfamily11 2024-11-08T13:28:47,937 DEBUG [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:47,937 DEBUG [StoreOpener-223bf549b03e7990028b7927dd897a23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:47,938 INFO [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] regionserver.HStore(327): Store=99abbd965e76aa0c5fe5a1011e45a9ae/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:47,938 INFO [StoreOpener-223bf549b03e7990028b7927dd897a23-1 {}] regionserver.HStore(327): Store=223bf549b03e7990028b7927dd897a23/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:47,938 INFO [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:28:47,938 INFO [StoreOpener-223bf549b03e7990028b7927dd897a23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 223bf549b03e7990028b7927dd897a23 2024-11-08T13:28:47,940 INFO [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99abbd965e76aa0c5fe5a1011e45a9ae columnFamilyName colfamily21 2024-11-08T13:28:47,940 DEBUG [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:47,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=6 2024-11-08T13:28:47,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 4d5935497231430a7057c66d28a9d018, server=6ccfea20c990,39565,1731072524927 in 156 msec 2024-11-08T13:28:47,942 INFO [StoreOpener-223bf549b03e7990028b7927dd897a23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 223bf549b03e7990028b7927dd897a23 columnFamilyName colfamily21 2024-11-08T13:28:47,942 INFO [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] regionserver.HStore(327): Store=99abbd965e76aa0c5fe5a1011e45a9ae/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:47,942 DEBUG [StoreOpener-223bf549b03e7990028b7927dd897a23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:47,942 INFO [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:28:47,943 INFO [StoreOpener-223bf549b03e7990028b7927dd897a23-1 {}] regionserver.HStore(327): Store=223bf549b03e7990028b7927dd897a23/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:47,943 INFO [StoreOpener-223bf549b03e7990028b7927dd897a23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 223bf549b03e7990028b7927dd897a23 2024-11-08T13:28:47,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4d5935497231430a7057c66d28a9d018, ASSIGN in 410 msec 2024-11-08T13:28:47,945 INFO [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99abbd965e76aa0c5fe5a1011e45a9ae columnFamilyName colfamily31 2024-11-08T13:28:47,945 DEBUG [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:47,945 INFO [StoreOpener-223bf549b03e7990028b7927dd897a23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 223bf549b03e7990028b7927dd897a23 columnFamilyName colfamily31 2024-11-08T13:28:47,945 DEBUG [StoreOpener-223bf549b03e7990028b7927dd897a23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:47,945 INFO [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] regionserver.HStore(327): Store=99abbd965e76aa0c5fe5a1011e45a9ae/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:47,946 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:28:47,946 INFO [StoreOpener-223bf549b03e7990028b7927dd897a23-1 {}] regionserver.HStore(327): Store=223bf549b03e7990028b7927dd897a23/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:47,946 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 223bf549b03e7990028b7927dd897a23 2024-11-08T13:28:47,947 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:28:47,947 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/223bf549b03e7990028b7927dd897a23 2024-11-08T13:28:47,948 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:28:47,948 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/223bf549b03e7990028b7927dd897a23 2024-11-08T13:28:47,949 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:28:47,949 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 223bf549b03e7990028b7927dd897a23 2024-11-08T13:28:47,949 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:28:47,949 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 223bf549b03e7990028b7927dd897a23 2024-11-08T13:28:47,950 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:47,950 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:47,951 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:28:47,952 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 223bf549b03e7990028b7927dd897a23 2024-11-08T13:28:47,955 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/223bf549b03e7990028b7927dd897a23/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:47,955 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:47,956 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 223bf549b03e7990028b7927dd897a23; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67065275, jitterRate=-6.495267152786255E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:47,956 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 223bf549b03e7990028b7927dd897a23 2024-11-08T13:28:47,956 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 223bf549b03e7990028b7927dd897a23: Running coprocessor pre-open hook at 1731072527933Writing region info on filesystem at 1731072527933Initializing all the Stores at 1731072527935 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072527935Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072527935Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072527935Cleaning up temporary data from old regions at 1731072527949 (+14 ms)Running coprocessor post-open hooks at 1731072527956 (+7 ms)Region opened successfully at 1731072527956 2024-11-08T13:28:47,957 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 99abbd965e76aa0c5fe5a1011e45a9ae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60428251, jitterRate=-0.09954889118671417}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:47,957 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:28:47,957 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 99abbd965e76aa0c5fe5a1011e45a9ae: Running coprocessor pre-open hook at 1731072527932Writing region info on filesystem at 1731072527932Initializing all the Stores at 1731072527934 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072527934Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072527934Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072527934Cleaning up temporary data from old regions at 1731072527949 (+15 ms)Running coprocessor post-open hooks at 1731072527957 (+8 ms)Region opened successfully at 1731072527957 2024-11-08T13:28:47,958 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23., pid=42, masterSystemTime=1731072527922 2024-11-08T13:28:47,958 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae., pid=31, masterSystemTime=1731072527863 2024-11-08T13:28:47,961 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23. 2024-11-08T13:28:47,961 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23. 2024-11-08T13:28:47,961 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c. 2024-11-08T13:28:47,961 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 0b082e035d4bc82dc2f93642d66c000c, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-08T13:28:47,962 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=223bf549b03e7990028b7927dd897a23, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:28:47,962 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 0b082e035d4bc82dc2f93642d66c000c 2024-11-08T13:28:47,962 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,962 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae. 2024-11-08T13:28:47,962 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae. 2024-11-08T13:28:47,962 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 0b082e035d4bc82dc2f93642d66c000c 2024-11-08T13:28:47,962 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 0b082e035d4bc82dc2f93642d66c000c 2024-11-08T13:28:47,962 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4. 2024-11-08T13:28:47,963 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 00f5b98ab61f3475c8968b8a7a9f6cb4, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-08T13:28:47,963 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:28:47,963 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:47,963 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:28:47,963 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:28:47,965 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 223bf549b03e7990028b7927dd897a23, server=6ccfea20c990,35367,1731072525051 because future has completed 2024-11-08T13:28:47,965 INFO [StoreOpener-0b082e035d4bc82dc2f93642d66c000c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 0b082e035d4bc82dc2f93642d66c000c 2024-11-08T13:28:47,965 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=99abbd965e76aa0c5fe5a1011e45a9ae, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,39565,1731072524927 2024-11-08T13:28:47,966 INFO [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:28:47,968 INFO [StoreOpener-0b082e035d4bc82dc2f93642d66c000c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b082e035d4bc82dc2f93642d66c000c columnFamilyName colfamily11 2024-11-08T13:28:47,968 INFO [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00f5b98ab61f3475c8968b8a7a9f6cb4 columnFamilyName colfamily11 2024-11-08T13:28:47,968 DEBUG [StoreOpener-0b082e035d4bc82dc2f93642d66c000c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:47,968 DEBUG [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:47,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99abbd965e76aa0c5fe5a1011e45a9ae, server=6ccfea20c990,39565,1731072524927 because future has completed 2024-11-08T13:28:47,969 INFO [StoreOpener-0b082e035d4bc82dc2f93642d66c000c-1 {}] regionserver.HStore(327): Store=0b082e035d4bc82dc2f93642d66c000c/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:47,969 INFO [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] regionserver.HStore(327): Store=00f5b98ab61f3475c8968b8a7a9f6cb4/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:47,969 INFO [StoreOpener-0b082e035d4bc82dc2f93642d66c000c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 0b082e035d4bc82dc2f93642d66c000c 2024-11-08T13:28:47,970 INFO [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:28:47,972 INFO [StoreOpener-0b082e035d4bc82dc2f93642d66c000c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b082e035d4bc82dc2f93642d66c000c columnFamilyName colfamily21 2024-11-08T13:28:47,972 INFO [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00f5b98ab61f3475c8968b8a7a9f6cb4 columnFamilyName colfamily21 2024-11-08T13:28:47,972 DEBUG [StoreOpener-0b082e035d4bc82dc2f93642d66c000c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:47,972 DEBUG [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:47,975 INFO [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] regionserver.HStore(327): Store=00f5b98ab61f3475c8968b8a7a9f6cb4/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:47,975 INFO [StoreOpener-0b082e035d4bc82dc2f93642d66c000c-1 {}] regionserver.HStore(327): Store=0b082e035d4bc82dc2f93642d66c000c/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:47,975 INFO [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:28:47,975 INFO [StoreOpener-0b082e035d4bc82dc2f93642d66c000c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 0b082e035d4bc82dc2f93642d66c000c 2024-11-08T13:28:47,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=23 2024-11-08T13:28:47,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 223bf549b03e7990028b7927dd897a23, server=6ccfea20c990,35367,1731072525051 in 172 msec 2024-11-08T13:28:47,977 INFO [StoreOpener-0b082e035d4bc82dc2f93642d66c000c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b082e035d4bc82dc2f93642d66c000c columnFamilyName colfamily31 2024-11-08T13:28:47,977 INFO [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00f5b98ab61f3475c8968b8a7a9f6cb4 columnFamilyName colfamily31 2024-11-08T13:28:47,977 DEBUG [StoreOpener-0b082e035d4bc82dc2f93642d66c000c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:47,977 DEBUG [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:47,978 INFO [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] regionserver.HStore(327): Store=00f5b98ab61f3475c8968b8a7a9f6cb4/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:47,979 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:28:47,979 INFO [StoreOpener-0b082e035d4bc82dc2f93642d66c000c-1 {}] regionserver.HStore(327): Store=0b082e035d4bc82dc2f93642d66c000c/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:47,979 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 0b082e035d4bc82dc2f93642d66c000c 2024-11-08T13:28:47,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=223bf549b03e7990028b7927dd897a23, ASSIGN in 445 msec 2024-11-08T13:28:47,980 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0b082e035d4bc82dc2f93642d66c000c 2024-11-08T13:28:47,980 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:28:47,981 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0b082e035d4bc82dc2f93642d66c000c 2024-11-08T13:28:47,982 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:28:47,982 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 0b082e035d4bc82dc2f93642d66c000c 2024-11-08T13:28:47,982 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 0b082e035d4bc82dc2f93642d66c000c 2024-11-08T13:28:47,983 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:47,984 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:28:47,984 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:28:47,985 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:47,986 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 0b082e035d4bc82dc2f93642d66c000c 2024-11-08T13:28:47,987 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:28:47,989 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T13:28:47,990 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:47,990 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0b082e035d4bc82dc2f93642d66c000c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:47,991 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 00f5b98ab61f3475c8968b8a7a9f6cb4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73183949, jitterRate=0.09052582085132599}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:47,991 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 0b082e035d4bc82dc2f93642d66c000c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70997179, jitterRate=0.05794040858745575}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:47,991 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:28:47,991 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0b082e035d4bc82dc2f93642d66c000c 2024-11-08T13:28:47,991 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 00f5b98ab61f3475c8968b8a7a9f6cb4: Running coprocessor pre-open hook at 1731072527964Writing region info on filesystem at 1731072527964Initializing all the Stores at 1731072527965 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072527965Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072527965Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072527965Cleaning up temporary data from old regions at 1731072527984 (+19 ms)Running coprocessor post-open hooks at 1731072527991 (+7 ms)Region opened successfully at 1731072527991 2024-11-08T13:28:47,991 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 0b082e035d4bc82dc2f93642d66c000c: Running coprocessor pre-open hook at 1731072527963Writing region info on filesystem at 1731072527963Initializing all the Stores at 1731072527964 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072527964Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072527965 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072527965Cleaning up temporary data from old regions at 1731072527983 (+18 ms)Running coprocessor post-open hooks at 1731072527991 (+8 ms)Region opened successfully at 1731072527991 2024-11-08T13:28:47,994 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4., pid=36, masterSystemTime=1731072527863 2024-11-08T13:28:47,994 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c., pid=46, masterSystemTime=1731072527922 2024-11-08T13:28:47,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48043, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T13:28:47,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-08T13:28:47,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 99abbd965e76aa0c5fe5a1011e45a9ae, server=6ccfea20c990,39565,1731072524927 in 275 msec 2024-11-08T13:28:47,999 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4. 2024-11-08T13:28:48,000 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4. 2024-11-08T13:28:48,000 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe. 2024-11-08T13:28:48,000 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => c4e1216db90b6f396355f2b6bc308ffe, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-08T13:28:48,001 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:28:48,001 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:48,001 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:28:48,001 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:28:48,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=99abbd965e76aa0c5fe5a1011e45a9ae, ASSIGN in 464 msec 2024-11-08T13:28:48,002 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c. 2024-11-08T13:28:48,002 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c. 2024-11-08T13:28:48,003 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203. 2024-11-08T13:28:48,003 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => b7f58040b9958dcdcc78f264a8778203, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-08T13:28:48,003 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true b7f58040b9958dcdcc78f264a8778203 2024-11-08T13:28:48,003 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:48,003 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for b7f58040b9958dcdcc78f264a8778203 2024-11-08T13:28:48,003 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for b7f58040b9958dcdcc78f264a8778203 2024-11-08T13:28:48,004 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=0b082e035d4bc82dc2f93642d66c000c, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:28:48,005 INFO [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:28:48,006 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef. 2024-11-08T13:28:48,007 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 5ab60b8cfddafd2fcce990e602b31aef, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-08T13:28:48,007 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 5ab60b8cfddafd2fcce990e602b31aef 2024-11-08T13:28:48,007 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:48,007 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 5ab60b8cfddafd2fcce990e602b31aef 2024-11-08T13:28:48,008 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 5ab60b8cfddafd2fcce990e602b31aef 2024-11-08T13:28:48,009 INFO [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4e1216db90b6f396355f2b6bc308ffe columnFamilyName colfamily11 2024-11-08T13:28:48,009 DEBUG [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b082e035d4bc82dc2f93642d66c000c, server=6ccfea20c990,35367,1731072525051 because future has completed 2024-11-08T13:28:48,010 INFO [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] regionserver.HStore(327): Store=c4e1216db90b6f396355f2b6bc308ffe/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,010 INFO [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:28:48,011 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=00f5b98ab61f3475c8968b8a7a9f6cb4, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,39565,1731072524927 2024-11-08T13:28:48,013 INFO [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4e1216db90b6f396355f2b6bc308ffe columnFamilyName colfamily21 2024-11-08T13:28:48,013 DEBUG [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,013 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40009 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=6ccfea20c990,39565,1731072524927, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=00f5b98ab61f3475c8968b8a7a9f6cb4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T13:28:48,014 INFO [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] regionserver.HStore(327): Store=c4e1216db90b6f396355f2b6bc308ffe/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,014 INFO [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:28:48,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00f5b98ab61f3475c8968b8a7a9f6cb4, server=6ccfea20c990,39565,1731072524927 because future has completed 2024-11-08T13:28:48,016 INFO [StoreOpener-b7f58040b9958dcdcc78f264a8778203-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region b7f58040b9958dcdcc78f264a8778203 2024-11-08T13:28:48,016 INFO [StoreOpener-5ab60b8cfddafd2fcce990e602b31aef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 5ab60b8cfddafd2fcce990e602b31aef 2024-11-08T13:28:48,017 INFO [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4e1216db90b6f396355f2b6bc308ffe columnFamilyName colfamily31 2024-11-08T13:28:48,017 DEBUG [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,019 INFO [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] regionserver.HStore(327): Store=c4e1216db90b6f396355f2b6bc308ffe/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,019 INFO [StoreOpener-5ab60b8cfddafd2fcce990e602b31aef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ab60b8cfddafd2fcce990e602b31aef columnFamilyName colfamily11 2024-11-08T13:28:48,019 DEBUG [StoreOpener-5ab60b8cfddafd2fcce990e602b31aef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,019 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:28:48,020 INFO [StoreOpener-b7f58040b9958dcdcc78f264a8778203-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7f58040b9958dcdcc78f264a8778203 columnFamilyName colfamily11 2024-11-08T13:28:48,020 DEBUG [StoreOpener-b7f58040b9958dcdcc78f264a8778203-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,020 INFO [StoreOpener-5ab60b8cfddafd2fcce990e602b31aef-1 {}] regionserver.HStore(327): Store=5ab60b8cfddafd2fcce990e602b31aef/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,021 INFO [StoreOpener-5ab60b8cfddafd2fcce990e602b31aef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 5ab60b8cfddafd2fcce990e602b31aef 2024-11-08T13:28:48,021 INFO [StoreOpener-b7f58040b9958dcdcc78f264a8778203-1 {}] regionserver.HStore(327): Store=b7f58040b9958dcdcc78f264a8778203/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,021 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:28:48,021 INFO [StoreOpener-b7f58040b9958dcdcc78f264a8778203-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region b7f58040b9958dcdcc78f264a8778203 2024-11-08T13:28:48,022 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:28:48,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=18 2024-11-08T13:28:48,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 0b082e035d4bc82dc2f93642d66c000c, server=6ccfea20c990,35367,1731072525051 in 194 msec 2024-11-08T13:28:48,023 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:28:48,024 INFO [StoreOpener-b7f58040b9958dcdcc78f264a8778203-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7f58040b9958dcdcc78f264a8778203 columnFamilyName colfamily21 2024-11-08T13:28:48,024 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:28:48,024 DEBUG [StoreOpener-b7f58040b9958dcdcc78f264a8778203-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,024 INFO [StoreOpener-5ab60b8cfddafd2fcce990e602b31aef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ab60b8cfddafd2fcce990e602b31aef columnFamilyName colfamily21 2024-11-08T13:28:48,024 DEBUG [StoreOpener-5ab60b8cfddafd2fcce990e602b31aef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,025 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:48,025 INFO [StoreOpener-b7f58040b9958dcdcc78f264a8778203-1 {}] regionserver.HStore(327): Store=b7f58040b9958dcdcc78f264a8778203/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,025 INFO [StoreOpener-5ab60b8cfddafd2fcce990e602b31aef-1 {}] regionserver.HStore(327): Store=5ab60b8cfddafd2fcce990e602b31aef/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,025 INFO [StoreOpener-b7f58040b9958dcdcc78f264a8778203-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region b7f58040b9958dcdcc78f264a8778203 2024-11-08T13:28:48,025 INFO [StoreOpener-5ab60b8cfddafd2fcce990e602b31aef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 5ab60b8cfddafd2fcce990e602b31aef 2024-11-08T13:28:48,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0b082e035d4bc82dc2f93642d66c000c, ASSIGN in 491 msec 2024-11-08T13:28:48,027 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:28:48,027 INFO [StoreOpener-5ab60b8cfddafd2fcce990e602b31aef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ab60b8cfddafd2fcce990e602b31aef columnFamilyName colfamily31 2024-11-08T13:28:48,028 DEBUG [StoreOpener-5ab60b8cfddafd2fcce990e602b31aef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,028 INFO [StoreOpener-b7f58040b9958dcdcc78f264a8778203-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b7f58040b9958dcdcc78f264a8778203 columnFamilyName colfamily31 2024-11-08T13:28:48,028 DEBUG [StoreOpener-b7f58040b9958dcdcc78f264a8778203-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,028 INFO [StoreOpener-5ab60b8cfddafd2fcce990e602b31aef-1 {}] regionserver.HStore(327): Store=5ab60b8cfddafd2fcce990e602b31aef/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,029 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 5ab60b8cfddafd2fcce990e602b31aef 2024-11-08T13:28:48,029 INFO [StoreOpener-b7f58040b9958dcdcc78f264a8778203-1 {}] regionserver.HStore(327): Store=b7f58040b9958dcdcc78f264a8778203/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,029 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for b7f58040b9958dcdcc78f264a8778203 2024-11-08T13:28:48,030 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ab60b8cfddafd2fcce990e602b31aef 2024-11-08T13:28:48,030 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b7f58040b9958dcdcc78f264a8778203 2024-11-08T13:28:48,031 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ab60b8cfddafd2fcce990e602b31aef 2024-11-08T13:28:48,031 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b7f58040b9958dcdcc78f264a8778203 2024-11-08T13:28:48,031 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:48,032 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened c4e1216db90b6f396355f2b6bc308ffe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65481798, jitterRate=-0.02424517273902893}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:48,032 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:28:48,032 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for c4e1216db90b6f396355f2b6bc308ffe: Running coprocessor pre-open hook at 1731072528001Writing region info on filesystem at 1731072528001Initializing all the Stores at 1731072528004 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528005 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528005Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528005Cleaning up temporary data from old regions at 1731072528024 (+19 ms)Running coprocessor post-open hooks at 1731072528032 (+8 ms)Region opened successfully at 1731072528032 2024-11-08T13:28:48,033 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for b7f58040b9958dcdcc78f264a8778203 2024-11-08T13:28:48,033 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for b7f58040b9958dcdcc78f264a8778203 2024-11-08T13:28:48,033 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 5ab60b8cfddafd2fcce990e602b31aef 2024-11-08T13:28:48,033 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe., pid=37, masterSystemTime=1731072527863 2024-11-08T13:28:48,033 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 5ab60b8cfddafd2fcce990e602b31aef 2024-11-08T13:28:48,034 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:48,035 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:48,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=26 2024-11-08T13:28:48,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 00f5b98ab61f3475c8968b8a7a9f6cb4, server=6ccfea20c990,39565,1731072524927 in 281 msec 2024-11-08T13:28:48,037 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 5ab60b8cfddafd2fcce990e602b31aef 2024-11-08T13:28:48,037 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for b7f58040b9958dcdcc78f264a8778203 2024-11-08T13:28:48,037 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe. 2024-11-08T13:28:48,038 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe. 2024-11-08T13:28:48,038 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3. 2024-11-08T13:28:48,038 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 06547784f6904a15d1d127f091988ed3, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-08T13:28:48,038 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 06547784f6904a15d1d127f091988ed3 2024-11-08T13:28:48,039 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:48,039 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 06547784f6904a15d1d127f091988ed3 2024-11-08T13:28:48,039 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 06547784f6904a15d1d127f091988ed3 2024-11-08T13:28:48,042 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ab60b8cfddafd2fcce990e602b31aef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:48,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=00f5b98ab61f3475c8968b8a7a9f6cb4, ASSIGN in 504 msec 2024-11-08T13:28:48,042 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 5ab60b8cfddafd2fcce990e602b31aef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63050533, jitterRate=-0.06047384440898895}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:48,043 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5ab60b8cfddafd2fcce990e602b31aef 2024-11-08T13:28:48,043 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 5ab60b8cfddafd2fcce990e602b31aef: Running coprocessor pre-open hook at 1731072528008Writing region info on filesystem at 1731072528008Initializing all the Stores at 1731072528011 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528011Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528016 (+5 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528016Cleaning up temporary data from old regions at 1731072528034 (+18 ms)Running coprocessor post-open hooks at 1731072528043 (+9 ms)Region opened successfully at 1731072528043 2024-11-08T13:28:48,044 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=c4e1216db90b6f396355f2b6bc308ffe, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,39565,1731072524927 2024-11-08T13:28:48,044 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef., pid=52, masterSystemTime=1731072527989 2024-11-08T13:28:48,047 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4e1216db90b6f396355f2b6bc308ffe, server=6ccfea20c990,39565,1731072524927 because future has completed 2024-11-08T13:28:48,047 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef. 2024-11-08T13:28:48,047 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef. 2024-11-08T13:28:48,048 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac. 2024-11-08T13:28:48,048 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 286367177ffe2bf304003f3b829326ac, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-08T13:28:48,048 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=5ab60b8cfddafd2fcce990e602b31aef, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:28:48,048 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 286367177ffe2bf304003f3b829326ac 2024-11-08T13:28:48,049 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:48,049 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 286367177ffe2bf304003f3b829326ac 2024-11-08T13:28:48,049 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 286367177ffe2bf304003f3b829326ac 2024-11-08T13:28:48,051 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b7f58040b9958dcdcc78f264a8778203/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:48,051 INFO [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 06547784f6904a15d1d127f091988ed3 2024-11-08T13:28:48,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ab60b8cfddafd2fcce990e602b31aef, server=6ccfea20c990,35061,1731072525012 because future has completed 2024-11-08T13:28:48,053 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened b7f58040b9958dcdcc78f264a8778203; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71788972, jitterRate=0.06973904371261597}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:48,053 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b7f58040b9958dcdcc78f264a8778203 2024-11-08T13:28:48,053 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for b7f58040b9958dcdcc78f264a8778203: Running coprocessor pre-open hook at 1731072528004Writing region info on filesystem at 1731072528004Initializing all the Stores at 1731072528006 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528006Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528015 (+9 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528016 (+1 ms)Cleaning up temporary data from old regions at 1731072528033 (+17 ms)Running coprocessor post-open hooks at 1731072528053 (+20 ms)Region opened successfully at 1731072528053 2024-11-08T13:28:48,054 INFO [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06547784f6904a15d1d127f091988ed3 columnFamilyName colfamily11 2024-11-08T13:28:48,055 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203., pid=38, masterSystemTime=1731072527922 2024-11-08T13:28:48,055 DEBUG [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,056 INFO [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] regionserver.HStore(327): Store=06547784f6904a15d1d127f091988ed3/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,056 INFO [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 06547784f6904a15d1d127f091988ed3 2024-11-08T13:28:48,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=11 2024-11-08T13:28:48,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure c4e1216db90b6f396355f2b6bc308ffe, server=6ccfea20c990,39565,1731072524927 in 293 msec 2024-11-08T13:28:48,058 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203. 2024-11-08T13:28:48,058 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203. 2024-11-08T13:28:48,058 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa. 2024-11-08T13:28:48,059 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 5503a38fb5ee20ab8673683ba1a1bcaa, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-08T13:28:48,059 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 5503a38fb5ee20ab8673683ba1a1bcaa 2024-11-08T13:28:48,059 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=b7f58040b9958dcdcc78f264a8778203, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:28:48,059 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:48,059 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 5503a38fb5ee20ab8673683ba1a1bcaa 2024-11-08T13:28:48,059 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 5503a38fb5ee20ab8673683ba1a1bcaa 2024-11-08T13:28:48,060 INFO [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06547784f6904a15d1d127f091988ed3 columnFamilyName colfamily21 2024-11-08T13:28:48,060 DEBUG [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,061 INFO [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] regionserver.HStore(327): Store=06547784f6904a15d1d127f091988ed3/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,061 INFO [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 06547784f6904a15d1d127f091988ed3 2024-11-08T13:28:48,063 INFO [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06547784f6904a15d1d127f091988ed3 columnFamilyName colfamily31 2024-11-08T13:28:48,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure b7f58040b9958dcdcc78f264a8778203, server=6ccfea20c990,35367,1731072525051 because future has completed 2024-11-08T13:28:48,064 DEBUG [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,064 INFO [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] regionserver.HStore(327): Store=06547784f6904a15d1d127f091988ed3/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,065 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 06547784f6904a15d1d127f091988ed3 2024-11-08T13:28:48,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c4e1216db90b6f396355f2b6bc308ffe, ASSIGN in 530 msec 2024-11-08T13:28:48,066 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3 2024-11-08T13:28:48,067 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3 2024-11-08T13:28:48,069 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 06547784f6904a15d1d127f091988ed3 2024-11-08T13:28:48,069 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 06547784f6904a15d1d127f091988ed3 2024-11-08T13:28:48,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=25 2024-11-08T13:28:48,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure b7f58040b9958dcdcc78f264a8778203, server=6ccfea20c990,35367,1731072525051 in 298 msec 2024-11-08T13:28:48,070 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:48,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b7f58040b9958dcdcc78f264a8778203, ASSIGN in 538 msec 2024-11-08T13:28:48,073 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 06547784f6904a15d1d127f091988ed3 2024-11-08T13:28:48,073 INFO [StoreOpener-286367177ffe2bf304003f3b829326ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 286367177ffe2bf304003f3b829326ac 2024-11-08T13:28:48,073 INFO [StoreOpener-5503a38fb5ee20ab8673683ba1a1bcaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 5503a38fb5ee20ab8673683ba1a1bcaa 2024-11-08T13:28:48,076 INFO [StoreOpener-5503a38fb5ee20ab8673683ba1a1bcaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5503a38fb5ee20ab8673683ba1a1bcaa columnFamilyName colfamily11 2024-11-08T13:28:48,076 DEBUG [StoreOpener-5503a38fb5ee20ab8673683ba1a1bcaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,077 INFO [StoreOpener-5503a38fb5ee20ab8673683ba1a1bcaa-1 {}] regionserver.HStore(327): Store=5503a38fb5ee20ab8673683ba1a1bcaa/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,077 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:48,077 INFO [StoreOpener-5503a38fb5ee20ab8673683ba1a1bcaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 5503a38fb5ee20ab8673683ba1a1bcaa 2024-11-08T13:28:48,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=19 2024-11-08T13:28:48,079 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 06547784f6904a15d1d127f091988ed3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72464232, jitterRate=0.07980120182037354}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:48,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 5ab60b8cfddafd2fcce990e602b31aef, server=6ccfea20c990,35061,1731072525012 in 217 msec 2024-11-08T13:28:48,079 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06547784f6904a15d1d127f091988ed3 2024-11-08T13:28:48,079 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 06547784f6904a15d1d127f091988ed3: Running coprocessor pre-open hook at 1731072528039Writing region info on filesystem at 1731072528039Initializing all the Stores at 1731072528041 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528041Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528050 (+9 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528050Cleaning up temporary data from old regions at 1731072528069 (+19 ms)Running coprocessor post-open hooks at 1731072528079 (+10 ms)Region opened successfully at 1731072528079 2024-11-08T13:28:48,080 INFO [StoreOpener-5503a38fb5ee20ab8673683ba1a1bcaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5503a38fb5ee20ab8673683ba1a1bcaa columnFamilyName colfamily21 2024-11-08T13:28:48,080 DEBUG [StoreOpener-5503a38fb5ee20ab8673683ba1a1bcaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,081 INFO [StoreOpener-5503a38fb5ee20ab8673683ba1a1bcaa-1 {}] regionserver.HStore(327): Store=5503a38fb5ee20ab8673683ba1a1bcaa/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,081 INFO [StoreOpener-5503a38fb5ee20ab8673683ba1a1bcaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 5503a38fb5ee20ab8673683ba1a1bcaa 2024-11-08T13:28:48,082 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3., pid=32, masterSystemTime=1731072527863 2024-11-08T13:28:48,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ab60b8cfddafd2fcce990e602b31aef, ASSIGN in 547 msec 2024-11-08T13:28:48,083 INFO [StoreOpener-286367177ffe2bf304003f3b829326ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 286367177ffe2bf304003f3b829326ac columnFamilyName colfamily11 2024-11-08T13:28:48,083 DEBUG [StoreOpener-286367177ffe2bf304003f3b829326ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,083 INFO [StoreOpener-5503a38fb5ee20ab8673683ba1a1bcaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5503a38fb5ee20ab8673683ba1a1bcaa columnFamilyName colfamily31 2024-11-08T13:28:48,084 INFO [StoreOpener-286367177ffe2bf304003f3b829326ac-1 {}] regionserver.HStore(327): Store=286367177ffe2bf304003f3b829326ac/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,084 DEBUG [StoreOpener-5503a38fb5ee20ab8673683ba1a1bcaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,084 INFO [StoreOpener-286367177ffe2bf304003f3b829326ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 286367177ffe2bf304003f3b829326ac 2024-11-08T13:28:48,085 INFO [StoreOpener-5503a38fb5ee20ab8673683ba1a1bcaa-1 {}] regionserver.HStore(327): Store=5503a38fb5ee20ab8673683ba1a1bcaa/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,086 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 5503a38fb5ee20ab8673683ba1a1bcaa 2024-11-08T13:28:48,087 INFO [StoreOpener-286367177ffe2bf304003f3b829326ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 286367177ffe2bf304003f3b829326ac columnFamilyName colfamily21 2024-11-08T13:28:48,087 DEBUG [StoreOpener-286367177ffe2bf304003f3b829326ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,087 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5503a38fb5ee20ab8673683ba1a1bcaa 2024-11-08T13:28:48,087 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3. 2024-11-08T13:28:48,087 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3. 2024-11-08T13:28:48,088 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd. 2024-11-08T13:28:48,088 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 5ef0046eead8230ad7213c161e3532bd, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-08T13:28:48,088 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5503a38fb5ee20ab8673683ba1a1bcaa 2024-11-08T13:28:48,088 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:28:48,088 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:48,088 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:28:48,088 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=06547784f6904a15d1d127f091988ed3, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,39565,1731072524927 2024-11-08T13:28:48,088 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:28:48,089 INFO [StoreOpener-286367177ffe2bf304003f3b829326ac-1 {}] regionserver.HStore(327): Store=286367177ffe2bf304003f3b829326ac/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,089 INFO [StoreOpener-286367177ffe2bf304003f3b829326ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 286367177ffe2bf304003f3b829326ac 2024-11-08T13:28:48,089 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 5503a38fb5ee20ab8673683ba1a1bcaa 2024-11-08T13:28:48,089 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 5503a38fb5ee20ab8673683ba1a1bcaa 2024-11-08T13:28:48,091 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:48,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06547784f6904a15d1d127f091988ed3, server=6ccfea20c990,39565,1731072524927 because future has completed 2024-11-08T13:28:48,093 INFO [StoreOpener-286367177ffe2bf304003f3b829326ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 286367177ffe2bf304003f3b829326ac columnFamilyName colfamily31 2024-11-08T13:28:48,093 DEBUG [StoreOpener-286367177ffe2bf304003f3b829326ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,094 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 5503a38fb5ee20ab8673683ba1a1bcaa 2024-11-08T13:28:48,094 INFO [StoreOpener-286367177ffe2bf304003f3b829326ac-1 {}] regionserver.HStore(327): Store=286367177ffe2bf304003f3b829326ac/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,095 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 286367177ffe2bf304003f3b829326ac 2024-11-08T13:28:48,095 INFO [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:28:48,096 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/286367177ffe2bf304003f3b829326ac 2024-11-08T13:28:48,097 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/286367177ffe2bf304003f3b829326ac 2024-11-08T13:28:48,097 INFO [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ef0046eead8230ad7213c161e3532bd columnFamilyName colfamily11 2024-11-08T13:28:48,097 DEBUG [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,099 INFO [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] regionserver.HStore(327): Store=5ef0046eead8230ad7213c161e3532bd/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,099 INFO [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:28:48,099 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5503a38fb5ee20ab8673683ba1a1bcaa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:48,100 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 286367177ffe2bf304003f3b829326ac 2024-11-08T13:28:48,101 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 286367177ffe2bf304003f3b829326ac 2024-11-08T13:28:48,101 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 5503a38fb5ee20ab8673683ba1a1bcaa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69698870, jitterRate=0.03859409689903259}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:48,101 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5503a38fb5ee20ab8673683ba1a1bcaa 2024-11-08T13:28:48,101 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 5503a38fb5ee20ab8673683ba1a1bcaa: Running coprocessor pre-open hook at 1731072528060Writing region info on filesystem at 1731072528060Initializing all the Stores at 1731072528062 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528062Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528073 (+11 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528073Cleaning up temporary data from old regions at 1731072528090 (+17 ms)Running coprocessor post-open hooks at 1731072528101 (+11 ms)Region opened successfully at 1731072528101 2024-11-08T13:28:48,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=20 2024-11-08T13:28:48,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 06547784f6904a15d1d127f091988ed3, server=6ccfea20c990,39565,1731072524927 in 375 msec 2024-11-08T13:28:48,102 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:48,102 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa., pid=45, masterSystemTime=1731072527922 2024-11-08T13:28:48,104 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 286367177ffe2bf304003f3b829326ac 2024-11-08T13:28:48,105 INFO [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ef0046eead8230ad7213c161e3532bd columnFamilyName colfamily21 2024-11-08T13:28:48,105 DEBUG [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,106 INFO [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] regionserver.HStore(327): Store=5ef0046eead8230ad7213c161e3532bd/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,106 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa. 2024-11-08T13:28:48,106 INFO [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:28:48,106 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa. 2024-11-08T13:28:48,106 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5. 2024-11-08T13:28:48,106 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 7e4837322188444dc48d5c8632c81ce5, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-08T13:28:48,107 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 7e4837322188444dc48d5c8632c81ce5 2024-11-08T13:28:48,107 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:48,107 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 7e4837322188444dc48d5c8632c81ce5 2024-11-08T13:28:48,107 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 7e4837322188444dc48d5c8632c81ce5 2024-11-08T13:28:48,108 INFO [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ef0046eead8230ad7213c161e3532bd columnFamilyName colfamily31 2024-11-08T13:28:48,108 DEBUG [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,109 INFO [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] regionserver.HStore(327): Store=5ef0046eead8230ad7213c161e3532bd/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06547784f6904a15d1d127f091988ed3, ASSIGN in 569 msec 2024-11-08T13:28:48,109 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:28:48,111 INFO [StoreOpener-7e4837322188444dc48d5c8632c81ce5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 7e4837322188444dc48d5c8632c81ce5 2024-11-08T13:28:48,111 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=5503a38fb5ee20ab8673683ba1a1bcaa, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:28:48,111 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:28:48,111 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:28:48,112 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/286367177ffe2bf304003f3b829326ac/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:48,113 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 286367177ffe2bf304003f3b829326ac; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70424985, jitterRate=0.049414053559303284}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:48,113 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 286367177ffe2bf304003f3b829326ac 2024-11-08T13:28:48,114 INFO [StoreOpener-7e4837322188444dc48d5c8632c81ce5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e4837322188444dc48d5c8632c81ce5 columnFamilyName colfamily11 2024-11-08T13:28:48,114 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 286367177ffe2bf304003f3b829326ac: Running coprocessor pre-open hook at 1731072528049Writing region info on filesystem at 1731072528049Initializing all the Stores at 1731072528053 (+4 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528053Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528073 (+20 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528073Cleaning up temporary data from old regions at 1731072528101 (+28 ms)Running coprocessor post-open hooks at 1731072528113 (+12 ms)Region opened successfully at 1731072528113 2024-11-08T13:28:48,114 DEBUG [StoreOpener-7e4837322188444dc48d5c8632c81ce5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5503a38fb5ee20ab8673683ba1a1bcaa, server=6ccfea20c990,35367,1731072525051 because future has completed 2024-11-08T13:28:48,115 INFO [StoreOpener-7e4837322188444dc48d5c8632c81ce5-1 {}] regionserver.HStore(327): Store=7e4837322188444dc48d5c8632c81ce5/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,115 INFO [StoreOpener-7e4837322188444dc48d5c8632c81ce5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 7e4837322188444dc48d5c8632c81ce5 2024-11-08T13:28:48,117 INFO [StoreOpener-7e4837322188444dc48d5c8632c81ce5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e4837322188444dc48d5c8632c81ce5 columnFamilyName colfamily21 2024-11-08T13:28:48,117 DEBUG [StoreOpener-7e4837322188444dc48d5c8632c81ce5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,118 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:28:48,118 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac., pid=55, masterSystemTime=1731072527989 2024-11-08T13:28:48,118 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:28:48,118 INFO [StoreOpener-7e4837322188444dc48d5c8632c81ce5-1 {}] regionserver.HStore(327): Store=7e4837322188444dc48d5c8632c81ce5/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,119 INFO [StoreOpener-7e4837322188444dc48d5c8632c81ce5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 7e4837322188444dc48d5c8632c81ce5 2024-11-08T13:28:48,122 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac. 2024-11-08T13:28:48,122 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:48,122 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac. 2024-11-08T13:28:48,122 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170. 2024-11-08T13:28:48,122 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 4fbf6f08f931761acf639331e9fd9170, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-08T13:28:48,123 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 4fbf6f08f931761acf639331e9fd9170 2024-11-08T13:28:48,123 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:48,123 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 4fbf6f08f931761acf639331e9fd9170 2024-11-08T13:28:48,123 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 4fbf6f08f931761acf639331e9fd9170 2024-11-08T13:28:48,124 INFO [StoreOpener-7e4837322188444dc48d5c8632c81ce5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7e4837322188444dc48d5c8632c81ce5 columnFamilyName colfamily31 2024-11-08T13:28:48,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=15 2024-11-08T13:28:48,124 DEBUG [StoreOpener-7e4837322188444dc48d5c8632c81ce5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 5503a38fb5ee20ab8673683ba1a1bcaa, server=6ccfea20c990,35367,1731072525051 in 302 msec 2024-11-08T13:28:48,125 INFO [StoreOpener-7e4837322188444dc48d5c8632c81ce5-1 {}] regionserver.HStore(327): Store=7e4837322188444dc48d5c8632c81ce5/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,125 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 7e4837322188444dc48d5c8632c81ce5 2024-11-08T13:28:48,125 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:28:48,126 INFO [StoreOpener-4fbf6f08f931761acf639331e9fd9170-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 4fbf6f08f931761acf639331e9fd9170 2024-11-08T13:28:48,126 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7e4837322188444dc48d5c8632c81ce5 2024-11-08T13:28:48,127 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7e4837322188444dc48d5c8632c81ce5 2024-11-08T13:28:48,128 INFO [StoreOpener-4fbf6f08f931761acf639331e9fd9170-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4fbf6f08f931761acf639331e9fd9170 columnFamilyName colfamily11 2024-11-08T13:28:48,128 DEBUG [StoreOpener-4fbf6f08f931761acf639331e9fd9170-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,129 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:48,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5503a38fb5ee20ab8673683ba1a1bcaa, ASSIGN in 592 msec 2024-11-08T13:28:48,130 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 7e4837322188444dc48d5c8632c81ce5 2024-11-08T13:28:48,130 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 7e4837322188444dc48d5c8632c81ce5 2024-11-08T13:28:48,130 INFO [StoreOpener-4fbf6f08f931761acf639331e9fd9170-1 {}] regionserver.HStore(327): Store=4fbf6f08f931761acf639331e9fd9170/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,130 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 5ef0046eead8230ad7213c161e3532bd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71431581, jitterRate=0.06441350281238556}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:48,130 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:28:48,130 INFO [StoreOpener-4fbf6f08f931761acf639331e9fd9170-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 4fbf6f08f931761acf639331e9fd9170 2024-11-08T13:28:48,130 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=286367177ffe2bf304003f3b829326ac, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:28:48,131 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 5ef0046eead8230ad7213c161e3532bd: Running coprocessor pre-open hook at 1731072528089Writing region info on filesystem at 1731072528089Initializing all the Stores at 1731072528093 (+4 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528094 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528094Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528094Cleaning up temporary data from old regions at 1731072528118 (+24 ms)Running coprocessor post-open hooks at 1731072528130 (+12 ms)Region opened successfully at 1731072528130 2024-11-08T13:28:48,132 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:48,133 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd., pid=34, masterSystemTime=1731072527863 2024-11-08T13:28:48,134 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 7e4837322188444dc48d5c8632c81ce5 2024-11-08T13:28:48,135 INFO [StoreOpener-4fbf6f08f931761acf639331e9fd9170-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4fbf6f08f931761acf639331e9fd9170 columnFamilyName colfamily21 2024-11-08T13:28:48,135 DEBUG [StoreOpener-4fbf6f08f931761acf639331e9fd9170-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 286367177ffe2bf304003f3b829326ac, server=6ccfea20c990,35061,1731072525012 because future has completed 2024-11-08T13:28:48,136 INFO [StoreOpener-4fbf6f08f931761acf639331e9fd9170-1 {}] regionserver.HStore(327): Store=4fbf6f08f931761acf639331e9fd9170/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,136 INFO [StoreOpener-4fbf6f08f931761acf639331e9fd9170-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 4fbf6f08f931761acf639331e9fd9170 2024-11-08T13:28:48,137 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd. 2024-11-08T13:28:48,138 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd. 2024-11-08T13:28:48,138 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe. 2024-11-08T13:28:48,138 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 7c8fbd70358d0bf3e3f164569432ecfe, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-08T13:28:48,138 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:28:48,138 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:48,139 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:28:48,139 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:28:48,139 INFO [StoreOpener-4fbf6f08f931761acf639331e9fd9170-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4fbf6f08f931761acf639331e9fd9170 columnFamilyName colfamily31 2024-11-08T13:28:48,139 DEBUG [StoreOpener-4fbf6f08f931761acf639331e9fd9170-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,140 INFO [StoreOpener-4fbf6f08f931761acf639331e9fd9170-1 {}] regionserver.HStore(327): Store=4fbf6f08f931761acf639331e9fd9170/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,140 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 4fbf6f08f931761acf639331e9fd9170 2024-11-08T13:28:48,141 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4fbf6f08f931761acf639331e9fd9170 2024-11-08T13:28:48,142 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4fbf6f08f931761acf639331e9fd9170 2024-11-08T13:28:48,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=12 2024-11-08T13:28:48,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 286367177ffe2bf304003f3b829326ac, server=6ccfea20c990,35061,1731072525012 in 266 msec 2024-11-08T13:28:48,143 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 4fbf6f08f931761acf639331e9fd9170 2024-11-08T13:28:48,143 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 4fbf6f08f931761acf639331e9fd9170 2024-11-08T13:28:48,144 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:48,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=286367177ffe2bf304003f3b829326ac, ASSIGN in 610 msec 2024-11-08T13:28:48,146 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7e4837322188444dc48d5c8632c81ce5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:48,147 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 4fbf6f08f931761acf639331e9fd9170 2024-11-08T13:28:48,147 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=5ef0046eead8230ad7213c161e3532bd, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,39565,1731072524927 2024-11-08T13:28:48,147 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 7e4837322188444dc48d5c8632c81ce5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74111924, jitterRate=0.10435372591018677}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:48,147 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7e4837322188444dc48d5c8632c81ce5 2024-11-08T13:28:48,147 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 7e4837322188444dc48d5c8632c81ce5: Running coprocessor pre-open hook at 1731072528107Writing region info on filesystem at 1731072528107Initializing all the Stores at 1731072528109 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528109Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528110 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528110Cleaning up temporary data from old regions at 1731072528130 (+20 ms)Running coprocessor post-open hooks at 1731072528147 (+17 ms)Region opened successfully at 1731072528147 2024-11-08T13:28:48,149 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5., pid=47, masterSystemTime=1731072527922 2024-11-08T13:28:48,149 INFO [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:28:48,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ef0046eead8230ad7213c161e3532bd, server=6ccfea20c990,39565,1731072524927 because future has completed 2024-11-08T13:28:48,153 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5. 2024-11-08T13:28:48,153 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5. 2024-11-08T13:28:48,153 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8. 2024-11-08T13:28:48,153 INFO [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c8fbd70358d0bf3e3f164569432ecfe columnFamilyName colfamily11 2024-11-08T13:28:48,153 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => d85397130af5b1fc184790336ef5d2b8, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-08T13:28:48,153 DEBUG [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,154 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true d85397130af5b1fc184790336ef5d2b8 2024-11-08T13:28:48,154 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:48,154 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for d85397130af5b1fc184790336ef5d2b8 2024-11-08T13:28:48,154 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for d85397130af5b1fc184790336ef5d2b8 2024-11-08T13:28:48,154 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4fbf6f08f931761acf639331e9fd9170/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:48,155 INFO [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] regionserver.HStore(327): Store=7c8fbd70358d0bf3e3f164569432ecfe/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,155 INFO [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:28:48,155 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 4fbf6f08f931761acf639331e9fd9170; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66719586, jitterRate=-0.005800694227218628}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:48,155 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4fbf6f08f931761acf639331e9fd9170 2024-11-08T13:28:48,155 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 4fbf6f08f931761acf639331e9fd9170: Running coprocessor pre-open hook at 1731072528123Writing region info on filesystem at 1731072528123Initializing all the Stores at 1731072528125 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528125Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528125Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528126 (+1 ms)Cleaning up temporary data from old regions at 1731072528143 (+17 ms)Running coprocessor post-open hooks at 1731072528155 (+12 ms)Region opened successfully at 1731072528155 2024-11-08T13:28:48,156 INFO [StoreOpener-d85397130af5b1fc184790336ef5d2b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region d85397130af5b1fc184790336ef5d2b8 2024-11-08T13:28:48,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=17 2024-11-08T13:28:48,157 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170., pid=50, masterSystemTime=1731072527989 2024-11-08T13:28:48,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 5ef0046eead8230ad7213c161e3532bd, server=6ccfea20c990,39565,1731072524927 in 416 msec 2024-11-08T13:28:48,158 INFO [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c8fbd70358d0bf3e3f164569432ecfe columnFamilyName colfamily21 2024-11-08T13:28:48,159 DEBUG [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,160 INFO [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] regionserver.HStore(327): Store=7c8fbd70358d0bf3e3f164569432ecfe/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,160 INFO [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:28:48,160 INFO [StoreOpener-d85397130af5b1fc184790336ef5d2b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d85397130af5b1fc184790336ef5d2b8 columnFamilyName colfamily11 2024-11-08T13:28:48,160 DEBUG [StoreOpener-d85397130af5b1fc184790336ef5d2b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,160 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170. 2024-11-08T13:28:48,161 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170. 2024-11-08T13:28:48,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ef0046eead8230ad7213c161e3532bd, ASSIGN in 625 msec 2024-11-08T13:28:48,161 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13. 2024-11-08T13:28:48,161 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => c02aa10a67704d3f15deae347223dc13, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-08T13:28:48,161 INFO [StoreOpener-d85397130af5b1fc184790336ef5d2b8-1 {}] regionserver.HStore(327): Store=d85397130af5b1fc184790336ef5d2b8/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,161 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true c02aa10a67704d3f15deae347223dc13 2024-11-08T13:28:48,162 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:48,162 INFO [StoreOpener-d85397130af5b1fc184790336ef5d2b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region d85397130af5b1fc184790336ef5d2b8 2024-11-08T13:28:48,162 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for c02aa10a67704d3f15deae347223dc13 2024-11-08T13:28:48,162 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for c02aa10a67704d3f15deae347223dc13 2024-11-08T13:28:48,164 INFO [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c8fbd70358d0bf3e3f164569432ecfe columnFamilyName colfamily31 2024-11-08T13:28:48,164 INFO [StoreOpener-c02aa10a67704d3f15deae347223dc13-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region c02aa10a67704d3f15deae347223dc13 2024-11-08T13:28:48,164 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=4fbf6f08f931761acf639331e9fd9170, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:28:48,164 DEBUG [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,164 INFO [StoreOpener-d85397130af5b1fc184790336ef5d2b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d85397130af5b1fc184790336ef5d2b8 columnFamilyName colfamily21 2024-11-08T13:28:48,164 DEBUG [StoreOpener-d85397130af5b1fc184790336ef5d2b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,164 INFO [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] regionserver.HStore(327): Store=7c8fbd70358d0bf3e3f164569432ecfe/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,165 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:28:48,165 INFO [StoreOpener-d85397130af5b1fc184790336ef5d2b8-1 {}] regionserver.HStore(327): Store=d85397130af5b1fc184790336ef5d2b8/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,165 INFO [StoreOpener-d85397130af5b1fc184790336ef5d2b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region d85397130af5b1fc184790336ef5d2b8 2024-11-08T13:28:48,168 INFO [StoreOpener-c02aa10a67704d3f15deae347223dc13-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c02aa10a67704d3f15deae347223dc13 columnFamilyName colfamily11 2024-11-08T13:28:48,168 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:28:48,168 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=7e4837322188444dc48d5c8632c81ce5, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:28:48,168 DEBUG [StoreOpener-c02aa10a67704d3f15deae347223dc13-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,169 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:28:48,169 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4fbf6f08f931761acf639331e9fd9170, server=6ccfea20c990,35061,1731072525012 because future has completed 2024-11-08T13:28:48,169 INFO [StoreOpener-d85397130af5b1fc184790336ef5d2b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d85397130af5b1fc184790336ef5d2b8 columnFamilyName colfamily31 2024-11-08T13:28:48,170 DEBUG [StoreOpener-d85397130af5b1fc184790336ef5d2b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,170 INFO [StoreOpener-c02aa10a67704d3f15deae347223dc13-1 {}] regionserver.HStore(327): Store=c02aa10a67704d3f15deae347223dc13/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,170 INFO [StoreOpener-c02aa10a67704d3f15deae347223dc13-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region c02aa10a67704d3f15deae347223dc13 2024-11-08T13:28:48,173 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:28:48,173 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:28:48,172 INFO [StoreOpener-d85397130af5b1fc184790336ef5d2b8-1 {}] regionserver.HStore(327): Store=d85397130af5b1fc184790336ef5d2b8/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,173 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for d85397130af5b1fc184790336ef5d2b8 2024-11-08T13:28:48,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7e4837322188444dc48d5c8632c81ce5, server=6ccfea20c990,35367,1731072525051 because future has completed 2024-11-08T13:28:48,174 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:48,174 INFO [StoreOpener-c02aa10a67704d3f15deae347223dc13-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c02aa10a67704d3f15deae347223dc13 columnFamilyName colfamily21 2024-11-08T13:28:48,174 DEBUG [StoreOpener-c02aa10a67704d3f15deae347223dc13-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,175 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d85397130af5b1fc184790336ef5d2b8 2024-11-08T13:28:48,175 INFO [StoreOpener-c02aa10a67704d3f15deae347223dc13-1 {}] regionserver.HStore(327): Store=c02aa10a67704d3f15deae347223dc13/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,176 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d85397130af5b1fc184790336ef5d2b8 2024-11-08T13:28:48,176 INFO [StoreOpener-c02aa10a67704d3f15deae347223dc13-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region c02aa10a67704d3f15deae347223dc13 2024-11-08T13:28:48,176 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:28:48,178 INFO [StoreOpener-c02aa10a67704d3f15deae347223dc13-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c02aa10a67704d3f15deae347223dc13 columnFamilyName colfamily31 2024-11-08T13:28:48,179 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for d85397130af5b1fc184790336ef5d2b8 2024-11-08T13:28:48,179 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for d85397130af5b1fc184790336ef5d2b8 2024-11-08T13:28:48,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=27 2024-11-08T13:28:48,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 4fbf6f08f931761acf639331e9fd9170, server=6ccfea20c990,35061,1731072525012 in 329 msec 2024-11-08T13:28:48,179 DEBUG [StoreOpener-c02aa10a67704d3f15deae347223dc13-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,180 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:48,181 INFO [StoreOpener-c02aa10a67704d3f15deae347223dc13-1 {}] regionserver.HStore(327): Store=c02aa10a67704d3f15deae347223dc13/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,181 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for c02aa10a67704d3f15deae347223dc13 2024-11-08T13:28:48,182 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:48,182 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for d85397130af5b1fc184790336ef5d2b8 2024-11-08T13:28:48,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4fbf6f08f931761acf639331e9fd9170, ASSIGN in 647 msec 2024-11-08T13:28:48,183 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c02aa10a67704d3f15deae347223dc13 2024-11-08T13:28:48,183 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c02aa10a67704d3f15deae347223dc13 2024-11-08T13:28:48,184 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 7c8fbd70358d0bf3e3f164569432ecfe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73461600, jitterRate=0.09466314315795898}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:48,184 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:28:48,184 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 7c8fbd70358d0bf3e3f164569432ecfe: Running coprocessor pre-open hook at 1731072528139Writing region info on filesystem at 1731072528139Initializing all the Stores at 1731072528140 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528140Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528149 (+9 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528149Cleaning up temporary data from old regions at 1731072528173 (+24 ms)Running coprocessor post-open hooks at 1731072528184 (+11 ms)Region opened successfully at 1731072528184 2024-11-08T13:28:48,186 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for c02aa10a67704d3f15deae347223dc13 2024-11-08T13:28:48,186 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for c02aa10a67704d3f15deae347223dc13 2024-11-08T13:28:48,187 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:48,187 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe., pid=41, masterSystemTime=1731072527863 2024-11-08T13:28:48,188 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d85397130af5b1fc184790336ef5d2b8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:48,189 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened d85397130af5b1fc184790336ef5d2b8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63152523, jitterRate=-0.05895407497882843}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:48,189 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d85397130af5b1fc184790336ef5d2b8 2024-11-08T13:28:48,189 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for d85397130af5b1fc184790336ef5d2b8: Running coprocessor pre-open hook at 1731072528154Writing region info on filesystem at 1731072528154Initializing all the Stores at 1731072528156 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528156Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528156Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528156Cleaning up temporary data from old regions at 1731072528179 (+23 ms)Running coprocessor post-open hooks at 1731072528189 (+10 ms)Region opened successfully at 1731072528189 2024-11-08T13:28:48,189 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for c02aa10a67704d3f15deae347223dc13 2024-11-08T13:28:48,191 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8., pid=43, masterSystemTime=1731072527922 2024-11-08T13:28:48,191 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe. 2024-11-08T13:28:48,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=7 2024-11-08T13:28:48,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 7e4837322188444dc48d5c8632c81ce5, server=6ccfea20c990,35367,1731072525051 in 355 msec 2024-11-08T13:28:48,191 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe. 2024-11-08T13:28:48,192 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57. 2024-11-08T13:28:48,192 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => fcf86fec97b560cab0f86ca4727e4e57, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-08T13:28:48,193 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:28:48,193 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:48,193 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:28:48,193 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:28:48,194 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8. 2024-11-08T13:28:48,195 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8. 2024-11-08T13:28:48,195 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e. 2024-11-08T13:28:48,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7e4837322188444dc48d5c8632c81ce5, ASSIGN in 660 msec 2024-11-08T13:28:48,195 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 50c49cba3ff42064b0e071b4b0d3656e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-08T13:28:48,196 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 50c49cba3ff42064b0e071b4b0d3656e 2024-11-08T13:28:48,196 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:48,196 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 50c49cba3ff42064b0e071b4b0d3656e 2024-11-08T13:28:48,196 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 50c49cba3ff42064b0e071b4b0d3656e 2024-11-08T13:28:48,196 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=d85397130af5b1fc184790336ef5d2b8, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:28:48,199 INFO [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:28:48,200 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c02aa10a67704d3f15deae347223dc13/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:48,202 INFO [StoreOpener-50c49cba3ff42064b0e071b4b0d3656e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 50c49cba3ff42064b0e071b4b0d3656e 2024-11-08T13:28:48,202 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened c02aa10a67704d3f15deae347223dc13; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75203179, jitterRate=0.12061469256877899}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:48,202 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c02aa10a67704d3f15deae347223dc13 2024-11-08T13:28:48,202 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for c02aa10a67704d3f15deae347223dc13: Running coprocessor pre-open hook at 1731072528162Writing region info on filesystem at 1731072528162Initializing all the Stores at 1731072528163 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528163Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528163Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528163Cleaning up temporary data from old regions at 1731072528186 (+23 ms)Running coprocessor post-open hooks at 1731072528202 (+16 ms)Region opened successfully at 1731072528202 2024-11-08T13:28:48,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure d85397130af5b1fc184790336ef5d2b8, server=6ccfea20c990,35367,1731072525051 because future has completed 2024-11-08T13:28:48,203 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=7c8fbd70358d0bf3e3f164569432ecfe, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,39565,1731072524927 2024-11-08T13:28:48,204 INFO [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fcf86fec97b560cab0f86ca4727e4e57 columnFamilyName colfamily11 2024-11-08T13:28:48,204 DEBUG [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,206 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13., pid=49, masterSystemTime=1731072527989 2024-11-08T13:28:48,207 INFO [StoreOpener-50c49cba3ff42064b0e071b4b0d3656e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50c49cba3ff42064b0e071b4b0d3656e columnFamilyName colfamily11 2024-11-08T13:28:48,207 DEBUG [StoreOpener-50c49cba3ff42064b0e071b4b0d3656e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,207 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c8fbd70358d0bf3e3f164569432ecfe, server=6ccfea20c990,39565,1731072524927 because future has completed 2024-11-08T13:28:48,207 INFO [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] regionserver.HStore(327): Store=fcf86fec97b560cab0f86ca4727e4e57/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,209 INFO [StoreOpener-50c49cba3ff42064b0e071b4b0d3656e-1 {}] regionserver.HStore(327): Store=50c49cba3ff42064b0e071b4b0d3656e/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,209 INFO [StoreOpener-50c49cba3ff42064b0e071b4b0d3656e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 50c49cba3ff42064b0e071b4b0d3656e 2024-11-08T13:28:48,209 INFO [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:28:48,210 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13. 2024-11-08T13:28:48,210 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13. 2024-11-08T13:28:48,210 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c. 2024-11-08T13:28:48,210 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => c6699738a1be7c8647568269e341753c, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-08T13:28:48,210 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true c6699738a1be7c8647568269e341753c 2024-11-08T13:28:48,211 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:48,211 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for c6699738a1be7c8647568269e341753c 2024-11-08T13:28:48,211 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for c6699738a1be7c8647568269e341753c 2024-11-08T13:28:48,212 INFO [StoreOpener-50c49cba3ff42064b0e071b4b0d3656e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50c49cba3ff42064b0e071b4b0d3656e columnFamilyName colfamily21 2024-11-08T13:28:48,212 DEBUG [StoreOpener-50c49cba3ff42064b0e071b4b0d3656e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,214 INFO [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fcf86fec97b560cab0f86ca4727e4e57 columnFamilyName colfamily21 2024-11-08T13:28:48,214 DEBUG [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,215 INFO [StoreOpener-50c49cba3ff42064b0e071b4b0d3656e-1 {}] regionserver.HStore(327): Store=50c49cba3ff42064b0e071b4b0d3656e/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,216 INFO [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] regionserver.HStore(327): Store=fcf86fec97b560cab0f86ca4727e4e57/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=10 2024-11-08T13:28:48,216 INFO [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:28:48,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure d85397130af5b1fc184790336ef5d2b8, server=6ccfea20c990,35367,1731072525051 in 401 msec 2024-11-08T13:28:48,217 INFO [StoreOpener-50c49cba3ff42064b0e071b4b0d3656e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 50c49cba3ff42064b0e071b4b0d3656e 2024-11-08T13:28:48,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d85397130af5b1fc184790336ef5d2b8, ASSIGN in 684 msec 2024-11-08T13:28:48,219 INFO [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fcf86fec97b560cab0f86ca4727e4e57 columnFamilyName colfamily31 2024-11-08T13:28:48,219 DEBUG [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,220 INFO [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] regionserver.HStore(327): Store=fcf86fec97b560cab0f86ca4727e4e57/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,220 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:28:48,221 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=c02aa10a67704d3f15deae347223dc13, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:28:48,221 INFO [StoreOpener-50c49cba3ff42064b0e071b4b0d3656e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 50c49cba3ff42064b0e071b4b0d3656e columnFamilyName colfamily31 2024-11-08T13:28:48,221 DEBUG [StoreOpener-50c49cba3ff42064b0e071b4b0d3656e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,222 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:28:48,222 INFO [StoreOpener-50c49cba3ff42064b0e071b4b0d3656e-1 {}] regionserver.HStore(327): Store=50c49cba3ff42064b0e071b4b0d3656e/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,223 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:28:48,223 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 50c49cba3ff42064b0e071b4b0d3656e 2024-11-08T13:28:48,224 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/50c49cba3ff42064b0e071b4b0d3656e 2024-11-08T13:28:48,226 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/50c49cba3ff42064b0e071b4b0d3656e 2024-11-08T13:28:48,226 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40009 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=6ccfea20c990,39565,1731072524927, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7c8fbd70358d0bf3e3f164569432ecfe. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T13:28:48,226 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:28:48,226 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:28:48,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure c02aa10a67704d3f15deae347223dc13, server=6ccfea20c990,35061,1731072525012 because future has completed 2024-11-08T13:28:48,227 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:48,228 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 50c49cba3ff42064b0e071b4b0d3656e 2024-11-08T13:28:48,228 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 50c49cba3ff42064b0e071b4b0d3656e 2024-11-08T13:28:48,229 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:48,229 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:28:48,231 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 50c49cba3ff42064b0e071b4b0d3656e 2024-11-08T13:28:48,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=8 2024-11-08T13:28:48,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 7c8fbd70358d0bf3e3f164569432ecfe, server=6ccfea20c990,39565,1731072524927 in 441 msec 2024-11-08T13:28:48,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7c8fbd70358d0bf3e3f164569432ecfe, ASSIGN in 702 msec 2024-11-08T13:28:48,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=21 2024-11-08T13:28:48,241 INFO [StoreOpener-c6699738a1be7c8647568269e341753c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region c6699738a1be7c8647568269e341753c 2024-11-08T13:28:48,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure c02aa10a67704d3f15deae347223dc13, server=6ccfea20c990,35061,1731072525012 in 397 msec 2024-11-08T13:28:48,243 INFO [StoreOpener-c6699738a1be7c8647568269e341753c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6699738a1be7c8647568269e341753c columnFamilyName colfamily11 2024-11-08T13:28:48,243 DEBUG [StoreOpener-c6699738a1be7c8647568269e341753c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,244 INFO [StoreOpener-c6699738a1be7c8647568269e341753c-1 {}] regionserver.HStore(327): Store=c6699738a1be7c8647568269e341753c/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c02aa10a67704d3f15deae347223dc13, ASSIGN in 709 msec 2024-11-08T13:28:48,244 INFO [StoreOpener-c6699738a1be7c8647568269e341753c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region c6699738a1be7c8647568269e341753c 2024-11-08T13:28:48,246 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:48,246 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened fcf86fec97b560cab0f86ca4727e4e57; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73648877, jitterRate=0.09745378792285919}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:48,247 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:28:48,247 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for fcf86fec97b560cab0f86ca4727e4e57: Running coprocessor pre-open hook at 1731072528194Writing region info on filesystem at 1731072528194Initializing all the Stores at 1731072528198 (+4 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528198Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528198Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528198Cleaning up temporary data from old regions at 1731072528226 (+28 ms)Running coprocessor post-open hooks at 1731072528247 (+21 ms)Region opened successfully at 1731072528247 2024-11-08T13:28:48,247 INFO [StoreOpener-c6699738a1be7c8647568269e341753c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6699738a1be7c8647568269e341753c columnFamilyName colfamily21 2024-11-08T13:28:48,247 DEBUG [StoreOpener-c6699738a1be7c8647568269e341753c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,249 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/50c49cba3ff42064b0e071b4b0d3656e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:48,249 INFO [StoreOpener-c6699738a1be7c8647568269e341753c-1 {}] regionserver.HStore(327): Store=c6699738a1be7c8647568269e341753c/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,249 INFO [StoreOpener-c6699738a1be7c8647568269e341753c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region c6699738a1be7c8647568269e341753c 2024-11-08T13:28:48,248 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57., pid=35, masterSystemTime=1731072527863 2024-11-08T13:28:48,250 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 50c49cba3ff42064b0e071b4b0d3656e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66950359, jitterRate=-0.0023619085550308228}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:48,250 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 50c49cba3ff42064b0e071b4b0d3656e 2024-11-08T13:28:48,251 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 50c49cba3ff42064b0e071b4b0d3656e: Running coprocessor pre-open hook at 1731072528196Writing region info on filesystem at 1731072528196Initializing all the Stores at 1731072528199 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528199Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528201 (+2 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528201Cleaning up temporary data from old regions at 1731072528228 (+27 ms)Running coprocessor post-open hooks at 1731072528250 (+22 ms)Region opened successfully at 1731072528250 2024-11-08T13:28:48,251 INFO [StoreOpener-c6699738a1be7c8647568269e341753c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6699738a1be7c8647568269e341753c columnFamilyName colfamily31 2024-11-08T13:28:48,252 DEBUG [StoreOpener-c6699738a1be7c8647568269e341753c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,252 INFO [StoreOpener-c6699738a1be7c8647568269e341753c-1 {}] regionserver.HStore(327): Store=c6699738a1be7c8647568269e341753c/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,253 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for c6699738a1be7c8647568269e341753c 2024-11-08T13:28:48,253 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57. 2024-11-08T13:28:48,253 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57. 2024-11-08T13:28:48,254 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c6699738a1be7c8647568269e341753c 2024-11-08T13:28:48,255 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c6699738a1be7c8647568269e341753c 2024-11-08T13:28:48,256 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for c6699738a1be7c8647568269e341753c 2024-11-08T13:28:48,256 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for c6699738a1be7c8647568269e341753c 2024-11-08T13:28:48,257 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:48,257 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e., pid=44, masterSystemTime=1731072527922 2024-11-08T13:28:48,260 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for c6699738a1be7c8647568269e341753c 2024-11-08T13:28:48,260 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=fcf86fec97b560cab0f86ca4727e4e57, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,39565,1731072524927 2024-11-08T13:28:48,261 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e. 2024-11-08T13:28:48,261 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e. 2024-11-08T13:28:48,261 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069. 2024-11-08T13:28:48,262 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => f973a563f81a494eeb1443f81f0bb069, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-08T13:28:48,262 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true f973a563f81a494eeb1443f81f0bb069 2024-11-08T13:28:48,262 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:48,263 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for f973a563f81a494eeb1443f81f0bb069 2024-11-08T13:28:48,263 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for f973a563f81a494eeb1443f81f0bb069 2024-11-08T13:28:48,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure fcf86fec97b560cab0f86ca4727e4e57, server=6ccfea20c990,39565,1731072524927 because future has completed 2024-11-08T13:28:48,265 INFO [StoreOpener-f973a563f81a494eeb1443f81f0bb069-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region f973a563f81a494eeb1443f81f0bb069 2024-11-08T13:28:48,265 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c6699738a1be7c8647568269e341753c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:48,267 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened c6699738a1be7c8647568269e341753c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59163770, jitterRate=-0.11839112639427185}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:48,267 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c6699738a1be7c8647568269e341753c 2024-11-08T13:28:48,267 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for c6699738a1be7c8647568269e341753c: Running coprocessor pre-open hook at 1731072528211Writing region info on filesystem at 1731072528211Initializing all the Stores at 1731072528213 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528213Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528229 (+16 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528229Cleaning up temporary data from old regions at 1731072528256 (+27 ms)Running coprocessor post-open hooks at 1731072528267 (+11 ms)Region opened successfully at 1731072528267 2024-11-08T13:28:48,267 INFO [StoreOpener-f973a563f81a494eeb1443f81f0bb069-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f973a563f81a494eeb1443f81f0bb069 columnFamilyName colfamily11 2024-11-08T13:28:48,267 DEBUG [StoreOpener-f973a563f81a494eeb1443f81f0bb069-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,269 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c., pid=53, masterSystemTime=1731072527989 2024-11-08T13:28:48,268 INFO [StoreOpener-f973a563f81a494eeb1443f81f0bb069-1 {}] regionserver.HStore(327): Store=f973a563f81a494eeb1443f81f0bb069/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,269 INFO [StoreOpener-f973a563f81a494eeb1443f81f0bb069-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region f973a563f81a494eeb1443f81f0bb069 2024-11-08T13:28:48,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40009 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T13:28:48,272 INFO [StoreOpener-f973a563f81a494eeb1443f81f0bb069-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f973a563f81a494eeb1443f81f0bb069 columnFamilyName colfamily21 2024-11-08T13:28:48,272 DEBUG [StoreOpener-f973a563f81a494eeb1443f81f0bb069-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,273 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c. 2024-11-08T13:28:48,273 INFO [StoreOpener-f973a563f81a494eeb1443f81f0bb069-1 {}] regionserver.HStore(327): Store=f973a563f81a494eeb1443f81f0bb069/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,273 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c. 2024-11-08T13:28:48,273 INFO [StoreOpener-f973a563f81a494eeb1443f81f0bb069-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region f973a563f81a494eeb1443f81f0bb069 2024-11-08T13:28:48,273 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b. 2024-11-08T13:28:48,274 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => e1c2472f75aa6a0ed9ba5017889bd71b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-08T13:28:48,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=22 2024-11-08T13:28:48,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure fcf86fec97b560cab0f86ca4727e4e57, server=6ccfea20c990,39565,1731072524927 in 527 msec 2024-11-08T13:28:48,274 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true e1c2472f75aa6a0ed9ba5017889bd71b 2024-11-08T13:28:48,274 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:48,275 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for e1c2472f75aa6a0ed9ba5017889bd71b 2024-11-08T13:28:48,275 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for e1c2472f75aa6a0ed9ba5017889bd71b 2024-11-08T13:28:48,275 INFO [StoreOpener-f973a563f81a494eeb1443f81f0bb069-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f973a563f81a494eeb1443f81f0bb069 columnFamilyName colfamily31 2024-11-08T13:28:48,275 DEBUG [StoreOpener-f973a563f81a494eeb1443f81f0bb069-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,276 INFO [StoreOpener-f973a563f81a494eeb1443f81f0bb069-1 {}] regionserver.HStore(327): Store=f973a563f81a494eeb1443f81f0bb069/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,276 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for f973a563f81a494eeb1443f81f0bb069 2024-11-08T13:28:48,277 INFO [StoreOpener-e1c2472f75aa6a0ed9ba5017889bd71b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region e1c2472f75aa6a0ed9ba5017889bd71b 2024-11-08T13:28:48,277 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f973a563f81a494eeb1443f81f0bb069 2024-11-08T13:28:48,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fcf86fec97b560cab0f86ca4727e4e57, ASSIGN in 742 msec 2024-11-08T13:28:48,278 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f973a563f81a494eeb1443f81f0bb069 2024-11-08T13:28:48,279 INFO [StoreOpener-e1c2472f75aa6a0ed9ba5017889bd71b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1c2472f75aa6a0ed9ba5017889bd71b columnFamilyName colfamily11 2024-11-08T13:28:48,279 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c6699738a1be7c8647568269e341753c, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:28:48,279 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for f973a563f81a494eeb1443f81f0bb069 2024-11-08T13:28:48,279 DEBUG [StoreOpener-e1c2472f75aa6a0ed9ba5017889bd71b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,280 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for f973a563f81a494eeb1443f81f0bb069 2024-11-08T13:28:48,280 INFO [StoreOpener-e1c2472f75aa6a0ed9ba5017889bd71b-1 {}] regionserver.HStore(327): Store=e1c2472f75aa6a0ed9ba5017889bd71b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,280 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:48,280 INFO [StoreOpener-e1c2472f75aa6a0ed9ba5017889bd71b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region e1c2472f75aa6a0ed9ba5017889bd71b 2024-11-08T13:28:48,283 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for f973a563f81a494eeb1443f81f0bb069 2024-11-08T13:28:48,283 INFO [StoreOpener-e1c2472f75aa6a0ed9ba5017889bd71b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1c2472f75aa6a0ed9ba5017889bd71b columnFamilyName colfamily21 2024-11-08T13:28:48,283 DEBUG [StoreOpener-e1c2472f75aa6a0ed9ba5017889bd71b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,283 INFO [StoreOpener-e1c2472f75aa6a0ed9ba5017889bd71b-1 {}] regionserver.HStore(327): Store=e1c2472f75aa6a0ed9ba5017889bd71b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,284 INFO [StoreOpener-e1c2472f75aa6a0ed9ba5017889bd71b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region e1c2472f75aa6a0ed9ba5017889bd71b 2024-11-08T13:28:48,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6699738a1be7c8647568269e341753c, server=6ccfea20c990,35061,1731072525012 because future has completed 2024-11-08T13:28:48,286 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f973a563f81a494eeb1443f81f0bb069/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:48,286 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=50c49cba3ff42064b0e071b4b0d3656e, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:28:48,286 INFO [StoreOpener-e1c2472f75aa6a0ed9ba5017889bd71b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1c2472f75aa6a0ed9ba5017889bd71b columnFamilyName colfamily31 2024-11-08T13:28:48,286 DEBUG [StoreOpener-e1c2472f75aa6a0ed9ba5017889bd71b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,287 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened f973a563f81a494eeb1443f81f0bb069; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67174764, jitterRate=9.819865226745605E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:48,287 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f973a563f81a494eeb1443f81f0bb069 2024-11-08T13:28:48,288 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for f973a563f81a494eeb1443f81f0bb069: Running coprocessor pre-open hook at 1731072528263Writing region info on filesystem at 1731072528263Initializing all the Stores at 1731072528264 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528264Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528265 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528265Cleaning up temporary data from old regions at 1731072528280 (+15 ms)Running coprocessor post-open hooks at 1731072528287 (+7 ms)Region opened successfully at 1731072528287 2024-11-08T13:28:48,288 INFO [StoreOpener-e1c2472f75aa6a0ed9ba5017889bd71b-1 {}] regionserver.HStore(327): Store=e1c2472f75aa6a0ed9ba5017889bd71b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,288 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for e1c2472f75aa6a0ed9ba5017889bd71b 2024-11-08T13:28:48,289 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069., pid=39, masterSystemTime=1731072527922 2024-11-08T13:28:48,291 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e1c2472f75aa6a0ed9ba5017889bd71b 2024-11-08T13:28:48,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 50c49cba3ff42064b0e071b4b0d3656e, server=6ccfea20c990,35367,1731072525051 because future has completed 2024-11-08T13:28:48,291 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e1c2472f75aa6a0ed9ba5017889bd71b 2024-11-08T13:28:48,292 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069. 2024-11-08T13:28:48,293 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069. 2024-11-08T13:28:48,293 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for e1c2472f75aa6a0ed9ba5017889bd71b 2024-11-08T13:28:48,293 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for e1c2472f75aa6a0ed9ba5017889bd71b 2024-11-08T13:28:48,294 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:48,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=5 2024-11-08T13:28:48,295 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for e1c2472f75aa6a0ed9ba5017889bd71b 2024-11-08T13:28:48,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c6699738a1be7c8647568269e341753c, server=6ccfea20c990,35061,1731072525012 in 430 msec 2024-11-08T13:28:48,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c6699738a1be7c8647568269e341753c, ASSIGN in 764 msec 2024-11-08T13:28:48,298 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e1c2472f75aa6a0ed9ba5017889bd71b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:48,299 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened e1c2472f75aa6a0ed9ba5017889bd71b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68541425, jitterRate=0.021346822381019592}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:48,299 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e1c2472f75aa6a0ed9ba5017889bd71b 2024-11-08T13:28:48,299 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for e1c2472f75aa6a0ed9ba5017889bd71b: Running coprocessor pre-open hook at 1731072528275Writing region info on filesystem at 1731072528275Initializing all the Stores at 1731072528276 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528277 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528277Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528277Cleaning up temporary data from old regions at 1731072528293 (+16 ms)Running coprocessor post-open hooks at 1731072528299 (+6 ms)Region opened successfully at 1731072528299 2024-11-08T13:28:48,300 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=f973a563f81a494eeb1443f81f0bb069, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:28:48,300 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b., pid=51, masterSystemTime=1731072527989 2024-11-08T13:28:48,303 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b. 2024-11-08T13:28:48,303 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b. 2024-11-08T13:28:48,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure f973a563f81a494eeb1443f81f0bb069, server=6ccfea20c990,35367,1731072525051 because future has completed 2024-11-08T13:28:48,304 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5. 2024-11-08T13:28:48,304 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 0f0799092106200541bc0c416b0bdbb5, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-08T13:28:48,304 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 0f0799092106200541bc0c416b0bdbb5 2024-11-08T13:28:48,304 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:48,304 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 0f0799092106200541bc0c416b0bdbb5 2024-11-08T13:28:48,304 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 0f0799092106200541bc0c416b0bdbb5 2024-11-08T13:28:48,307 INFO [StoreOpener-0f0799092106200541bc0c416b0bdbb5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 0f0799092106200541bc0c416b0bdbb5 2024-11-08T13:28:48,310 INFO [StoreOpener-0f0799092106200541bc0c416b0bdbb5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f0799092106200541bc0c416b0bdbb5 columnFamilyName colfamily11 2024-11-08T13:28:48,310 DEBUG [StoreOpener-0f0799092106200541bc0c416b0bdbb5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,311 INFO [StoreOpener-0f0799092106200541bc0c416b0bdbb5-1 {}] regionserver.HStore(327): Store=0f0799092106200541bc0c416b0bdbb5/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,311 INFO [StoreOpener-0f0799092106200541bc0c416b0bdbb5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 0f0799092106200541bc0c416b0bdbb5 2024-11-08T13:28:48,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=13 2024-11-08T13:28:48,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 50c49cba3ff42064b0e071b4b0d3656e, server=6ccfea20c990,35367,1731072525051 in 492 msec 2024-11-08T13:28:48,313 INFO [StoreOpener-0f0799092106200541bc0c416b0bdbb5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f0799092106200541bc0c416b0bdbb5 columnFamilyName colfamily21 2024-11-08T13:28:48,314 DEBUG [StoreOpener-0f0799092106200541bc0c416b0bdbb5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,314 INFO [StoreOpener-0f0799092106200541bc0c416b0bdbb5-1 {}] regionserver.HStore(327): Store=0f0799092106200541bc0c416b0bdbb5/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,315 INFO [StoreOpener-0f0799092106200541bc0c416b0bdbb5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 0f0799092106200541bc0c416b0bdbb5 2024-11-08T13:28:48,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=50c49cba3ff42064b0e071b4b0d3656e, ASSIGN in 780 msec 2024-11-08T13:28:48,319 INFO [StoreOpener-0f0799092106200541bc0c416b0bdbb5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f0799092106200541bc0c416b0bdbb5 columnFamilyName colfamily31 2024-11-08T13:28:48,319 DEBUG [StoreOpener-0f0799092106200541bc0c416b0bdbb5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,320 INFO [StoreOpener-0f0799092106200541bc0c416b0bdbb5-1 {}] regionserver.HStore(327): Store=0f0799092106200541bc0c416b0bdbb5/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,321 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 0f0799092106200541bc0c416b0bdbb5 2024-11-08T13:28:48,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=28 2024-11-08T13:28:48,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure f973a563f81a494eeb1443f81f0bb069, server=6ccfea20c990,35367,1731072525051 in 541 msec 2024-11-08T13:28:48,322 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0f0799092106200541bc0c416b0bdbb5 2024-11-08T13:28:48,323 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0f0799092106200541bc0c416b0bdbb5 2024-11-08T13:28:48,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f973a563f81a494eeb1443f81f0bb069, ASSIGN in 790 msec 2024-11-08T13:28:48,325 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 0f0799092106200541bc0c416b0bdbb5 2024-11-08T13:28:48,325 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 0f0799092106200541bc0c416b0bdbb5 2024-11-08T13:28:48,325 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:48,326 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=e1c2472f75aa6a0ed9ba5017889bd71b, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:28:48,331 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 0f0799092106200541bc0c416b0bdbb5 2024-11-08T13:28:48,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure e1c2472f75aa6a0ed9ba5017889bd71b, server=6ccfea20c990,35061,1731072525012 because future has completed 2024-11-08T13:28:48,336 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0f0799092106200541bc0c416b0bdbb5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:48,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=30 2024-11-08T13:28:48,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure e1c2472f75aa6a0ed9ba5017889bd71b, server=6ccfea20c990,35061,1731072525012 in 485 msec 2024-11-08T13:28:48,337 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 0f0799092106200541bc0c416b0bdbb5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59564963, jitterRate=-0.11241288483142853}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:48,337 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0f0799092106200541bc0c416b0bdbb5 2024-11-08T13:28:48,337 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 0f0799092106200541bc0c416b0bdbb5: Running coprocessor pre-open hook at 1731072528305Writing region info on filesystem at 1731072528305Initializing all the Stores at 1731072528306 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528306Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528307 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528307Cleaning up temporary data from old regions at 1731072528325 (+18 ms)Running coprocessor post-open hooks at 1731072528337 (+12 ms)Region opened successfully at 1731072528337 2024-11-08T13:28:48,338 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5., pid=54, masterSystemTime=1731072527989 2024-11-08T13:28:48,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e1c2472f75aa6a0ed9ba5017889bd71b, ASSIGN in 805 msec 2024-11-08T13:28:48,341 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5. 2024-11-08T13:28:48,341 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5. 2024-11-08T13:28:48,341 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c. 2024-11-08T13:28:48,341 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => e178bdc5865404fdcd9e707b6918569c, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-08T13:28:48,342 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=0f0799092106200541bc0c416b0bdbb5, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:28:48,342 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true e178bdc5865404fdcd9e707b6918569c 2024-11-08T13:28:48,342 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:48,342 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for e178bdc5865404fdcd9e707b6918569c 2024-11-08T13:28:48,342 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for e178bdc5865404fdcd9e707b6918569c 2024-11-08T13:28:48,344 INFO [StoreOpener-e178bdc5865404fdcd9e707b6918569c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region e178bdc5865404fdcd9e707b6918569c 2024-11-08T13:28:48,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0f0799092106200541bc0c416b0bdbb5, server=6ccfea20c990,35061,1731072525012 because future has completed 2024-11-08T13:28:48,347 INFO [StoreOpener-e178bdc5865404fdcd9e707b6918569c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e178bdc5865404fdcd9e707b6918569c columnFamilyName colfamily11 2024-11-08T13:28:48,347 DEBUG [StoreOpener-e178bdc5865404fdcd9e707b6918569c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,348 INFO [StoreOpener-e178bdc5865404fdcd9e707b6918569c-1 {}] regionserver.HStore(327): Store=e178bdc5865404fdcd9e707b6918569c/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,348 INFO [StoreOpener-e178bdc5865404fdcd9e707b6918569c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region e178bdc5865404fdcd9e707b6918569c 2024-11-08T13:28:48,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=9 2024-11-08T13:28:48,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 0f0799092106200541bc0c416b0bdbb5, server=6ccfea20c990,35061,1731072525012 in 480 msec 2024-11-08T13:28:48,350 INFO [StoreOpener-e178bdc5865404fdcd9e707b6918569c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e178bdc5865404fdcd9e707b6918569c columnFamilyName colfamily21 2024-11-08T13:28:48,350 DEBUG [StoreOpener-e178bdc5865404fdcd9e707b6918569c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,351 INFO [StoreOpener-e178bdc5865404fdcd9e707b6918569c-1 {}] regionserver.HStore(327): Store=e178bdc5865404fdcd9e707b6918569c/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,352 INFO [StoreOpener-e178bdc5865404fdcd9e707b6918569c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region e178bdc5865404fdcd9e707b6918569c 2024-11-08T13:28:48,353 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0f0799092106200541bc0c416b0bdbb5, ASSIGN in 818 msec 2024-11-08T13:28:48,354 INFO [StoreOpener-e178bdc5865404fdcd9e707b6918569c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e178bdc5865404fdcd9e707b6918569c columnFamilyName colfamily31 2024-11-08T13:28:48,354 DEBUG [StoreOpener-e178bdc5865404fdcd9e707b6918569c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,354 INFO [StoreOpener-e178bdc5865404fdcd9e707b6918569c-1 {}] regionserver.HStore(327): Store=e178bdc5865404fdcd9e707b6918569c/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,355 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for e178bdc5865404fdcd9e707b6918569c 2024-11-08T13:28:48,356 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e178bdc5865404fdcd9e707b6918569c 2024-11-08T13:28:48,356 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e178bdc5865404fdcd9e707b6918569c 2024-11-08T13:28:48,357 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for e178bdc5865404fdcd9e707b6918569c 2024-11-08T13:28:48,357 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for e178bdc5865404fdcd9e707b6918569c 2024-11-08T13:28:48,358 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:48,360 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for e178bdc5865404fdcd9e707b6918569c 2024-11-08T13:28:48,362 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e178bdc5865404fdcd9e707b6918569c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:48,363 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened e178bdc5865404fdcd9e707b6918569c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63686785, jitterRate=-0.050992950797080994}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:48,363 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e178bdc5865404fdcd9e707b6918569c 2024-11-08T13:28:48,363 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for e178bdc5865404fdcd9e707b6918569c: Running coprocessor pre-open hook at 1731072528342Writing region info on filesystem at 1731072528342Initializing all the Stores at 1731072528344 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528344Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528344Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528344Cleaning up temporary data from old regions at 1731072528358 (+14 ms)Running coprocessor post-open hooks at 1731072528363 (+5 ms)Region opened successfully at 1731072528363 2024-11-08T13:28:48,365 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c., pid=56, masterSystemTime=1731072527989 2024-11-08T13:28:48,368 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c. 2024-11-08T13:28:48,368 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c. 2024-11-08T13:28:48,368 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713. 2024-11-08T13:28:48,368 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 30f6256cfa21047086ecffea588e9713, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-08T13:28:48,368 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 30f6256cfa21047086ecffea588e9713 2024-11-08T13:28:48,369 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:28:48,369 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=e178bdc5865404fdcd9e707b6918569c, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:28:48,369 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 30f6256cfa21047086ecffea588e9713 2024-11-08T13:28:48,369 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 30f6256cfa21047086ecffea588e9713 2024-11-08T13:28:48,371 INFO [StoreOpener-30f6256cfa21047086ecffea588e9713-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 30f6256cfa21047086ecffea588e9713 2024-11-08T13:28:48,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure e178bdc5865404fdcd9e707b6918569c, server=6ccfea20c990,35061,1731072525012 because future has completed 2024-11-08T13:28:48,373 INFO [StoreOpener-30f6256cfa21047086ecffea588e9713-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30f6256cfa21047086ecffea588e9713 columnFamilyName colfamily11 2024-11-08T13:28:48,373 DEBUG [StoreOpener-30f6256cfa21047086ecffea588e9713-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,374 INFO [StoreOpener-30f6256cfa21047086ecffea588e9713-1 {}] regionserver.HStore(327): Store=30f6256cfa21047086ecffea588e9713/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,374 INFO [StoreOpener-30f6256cfa21047086ecffea588e9713-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 30f6256cfa21047086ecffea588e9713 2024-11-08T13:28:48,376 INFO [StoreOpener-30f6256cfa21047086ecffea588e9713-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30f6256cfa21047086ecffea588e9713 columnFamilyName colfamily21 2024-11-08T13:28:48,376 DEBUG [StoreOpener-30f6256cfa21047086ecffea588e9713-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=16 2024-11-08T13:28:48,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure e178bdc5865404fdcd9e707b6918569c, server=6ccfea20c990,35061,1731072525012 in 498 msec 2024-11-08T13:28:48,377 INFO [StoreOpener-30f6256cfa21047086ecffea588e9713-1 {}] regionserver.HStore(327): Store=30f6256cfa21047086ecffea588e9713/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,377 INFO [StoreOpener-30f6256cfa21047086ecffea588e9713-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 30f6256cfa21047086ecffea588e9713 2024-11-08T13:28:48,379 INFO [StoreOpener-30f6256cfa21047086ecffea588e9713-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30f6256cfa21047086ecffea588e9713 columnFamilyName colfamily31 2024-11-08T13:28:48,380 DEBUG [StoreOpener-30f6256cfa21047086ecffea588e9713-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:28:48,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e178bdc5865404fdcd9e707b6918569c, ASSIGN in 845 msec 2024-11-08T13:28:48,380 INFO [StoreOpener-30f6256cfa21047086ecffea588e9713-1 {}] regionserver.HStore(327): Store=30f6256cfa21047086ecffea588e9713/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:28:48,380 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 30f6256cfa21047086ecffea588e9713 2024-11-08T13:28:48,381 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/30f6256cfa21047086ecffea588e9713 2024-11-08T13:28:48,382 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/30f6256cfa21047086ecffea588e9713 2024-11-08T13:28:48,383 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 30f6256cfa21047086ecffea588e9713 2024-11-08T13:28:48,383 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 30f6256cfa21047086ecffea588e9713 2024-11-08T13:28:48,384 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:28:48,385 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 30f6256cfa21047086ecffea588e9713 2024-11-08T13:28:48,388 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/30f6256cfa21047086ecffea588e9713/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:28:48,388 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 30f6256cfa21047086ecffea588e9713; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69400811, jitterRate=0.034152671694755554}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:28:48,389 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 30f6256cfa21047086ecffea588e9713 2024-11-08T13:28:48,389 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 30f6256cfa21047086ecffea588e9713: Running coprocessor pre-open hook at 1731072528369Writing region info on filesystem at 1731072528369Initializing all the Stores at 1731072528370 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528370Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528371 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072528371Cleaning up temporary data from old regions at 1731072528383 (+12 ms)Running coprocessor post-open hooks at 1731072528389 (+6 ms)Region opened successfully at 1731072528389 2024-11-08T13:28:48,390 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713., pid=48, masterSystemTime=1731072527989 2024-11-08T13:28:48,392 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713. 2024-11-08T13:28:48,393 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713. 2024-11-08T13:28:48,393 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=30f6256cfa21047086ecffea588e9713, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:28:48,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30f6256cfa21047086ecffea588e9713, server=6ccfea20c990,35061,1731072525012 because future has completed 2024-11-08T13:28:48,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=24 2024-11-08T13:28:48,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 30f6256cfa21047086ecffea588e9713, server=6ccfea20c990,35061,1731072525012 in 561 msec 2024-11-08T13:28:48,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=4 2024-11-08T13:28:48,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=30f6256cfa21047086ecffea588e9713, ASSIGN in 869 msec 2024-11-08T13:28:48,406 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T13:28:48,406 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731072528406"}]},"ts":"1731072528406"} 2024-11-08T13:28:48,409 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, state=ENABLED in hbase:meta 2024-11-08T13:28:48,411 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T13:28:48,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true in 1.2940 sec 2024-11-08T13:28:49,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40009 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T13:28:49,282 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true get assigned. Timeout = 60000ms 2024-11-08T13:28:49,282 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true completed 2024-11-08T13:28:49,283 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:28:49,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39565 {}] regionserver.StoreScanner(1138): Switch to stream read (scanned=32802 bytes) of info 2024-11-08T13:28:49,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true assigned to meta. Checking AM states. 2024-11-08T13:28:49,310 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:28:49,310 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true assigned. 2024-11-08T13:28:49,353 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aab', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,356 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aaa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,358 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aac', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,359 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aad', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,360 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aae', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,361 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aaf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,362 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aag', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,363 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aah', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,443 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,445 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,447 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,448 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbe', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,449 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,499 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,501 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,503 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,504 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='cce', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,506 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,507 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='cch', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,546 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,549 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='dde', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,550 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,552 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,553 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,593 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eef', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,594 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eee', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,597 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eeg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,598 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eeh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,600 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eej', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,602 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eei', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,636 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,638 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='fff', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,640 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,642 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,643 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,645 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,681 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,683 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,685 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,687 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,689 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,726 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,728 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,730 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,732 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,733 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,735 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,778 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iii', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,781 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iik', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,783 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iij', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,785 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iil', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,786 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iim', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,787 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iin', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,788 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iio', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,821 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,824 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,826 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,828 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,830 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,832 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,866 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,869 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,871 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,874 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,876 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kko', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,878 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:49,916 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lll', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,919 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,921 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,923 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lln', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,928 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,931 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:49,959 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,962 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,964 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,967 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,969 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:49,972 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,003 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,006 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nno', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,009 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,011 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,013 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,015 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nns', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,055 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ooo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,057 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oop', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,060 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oor', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,062 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ooq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,065 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oos', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,067 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oot', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,104 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,107 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,108 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,111 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='pps', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,112 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,148 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,150 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,152 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqs', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,155 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,157 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,159 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,192 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,195 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrs', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,197 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rru', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,199 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,201 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,203 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,236 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sst', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,238 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sss', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,241 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,243 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,245 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,247 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,277 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,279 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,281 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,283 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,287 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='tty', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,289 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,323 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,325 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,327 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,329 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uux', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,331 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,333 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,367 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,369 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,371 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,373 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,378 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,380 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vwa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,410 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,412 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='www', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,415 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,416 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,419 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wxb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,421 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wxa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,456 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,458 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,460 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,462 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xya', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,464 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xyb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069., hostname=6ccfea20c990,35367,1731072525051, seqNum=2] 2024-11-08T13:28:50,512 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,515 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,517 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yza', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,519 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,520 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,522 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae., hostname=6ccfea20c990,39565,1731072524927, seqNum=2] 2024-11-08T13:28:50,553 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='zzz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b., hostname=6ccfea20c990,35061,1731072525012, seqNum=2] 2024-11-08T13:28:50,581 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T13:28:50,595 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T13:28:50,596 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T13:28:50,611 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T13:28:50,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35061 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 5625 Client: jenkins//172.17.0.2 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c. 2024-11-08T13:28:50,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35367 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 5624 Client: jenkins//172.17.0.2 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5. 2024-11-08T13:28:50,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39565 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 6327 Client: jenkins//172.17.0.2 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018. 2024-11-08T13:28:51,344 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,, stopping at row=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true ,, for max=2147483647 with caching=100 2024-11-08T13:28:51,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39565 {}] regionserver.StoreScanner(1138): Switch to stream read (scanned=32802 bytes) of info 2024-11-08T13:28:51,606 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: true 2024-11-08T13:28:51,606 INFO [Time-limited test {}] assignment.AssignmentTestingUtil(82): KILL REGION SERVER 6ccfea20c990,39565,1731072524927 2024-11-08T13:28:51,607 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 6ccfea20c990,39565,1731072524927 2024-11-08T13:28:51,607 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 6ccfea20c990,39565,1731072524927: Simulated kill ***** 2024-11-08T13:28:51,607 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-08T13:28:51,609 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-08T13:28:51,611 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-08T13:28:51,611 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-08T13:28:51,611 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-08T13:28:51,612 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 544199856 }, "NonHeapMemoryUsage": { "committed": 157745152, "init": 7667712, "max": -1, "used": 154701328 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "6ccfea20c990" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "6ccfea20c990" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "6ccfea20c990" } ] } 2024-11-08T13:28:51,614 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] master.MasterRpcServices(700): 6ccfea20c990,39565,1731072524927 reported a fatal error: ***** ABORTING region server 6ccfea20c990,39565,1731072524927: Simulated kill ***** 2024-11-08T13:28:51,616 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6ccfea20c990,39565,1731072524927' ***** 2024-11-08T13:28:51,617 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-08T13:28:51,617 INFO [RS:0;6ccfea20c990:39565 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T13:28:51,617 INFO [RS:0;6ccfea20c990:39565 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-08T13:28:51,617 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:28:51,617 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T13:28:51,617 INFO [RS:0;6ccfea20c990:39565 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-08T13:28:51,617 INFO [RS:0;6ccfea20c990:39565 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T13:28:51,618 INFO [RS:0;6ccfea20c990:39565 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;6ccfea20c990:39565. 2024-11-08T13:28:51,618 DEBUG [RS:0;6ccfea20c990:39565 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T13:28:51,623 DEBUG [RS:0;6ccfea20c990:39565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:28:51,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741836_1012 (size=54214) 2024-11-08T13:28:51,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741836_1012 (size=54214) 2024-11-08T13:28:51,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741836_1012 (size=54214) 2024-11-08T13:28:51,628 INFO [regionserver/6ccfea20c990:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T13:28:51,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741835_1011 (size=334920) 2024-11-08T13:28:51,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741835_1011 (size=334920) 2024-11-08T13:28:51,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741835_1011 (size=334920) 2024-11-08T13:28:51,634 DEBUG [RS:0;6ccfea20c990:39565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:28:51,634 INFO [RS:0;6ccfea20c990:39565 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T13:28:51,635 INFO [RS:0;6ccfea20c990:39565 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39565 2024-11-08T13:28:51,641 INFO [RS:0;6ccfea20c990:39565 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T13:28:51,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39565-0x1001f85d9a10001, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6ccfea20c990,39565,1731072524927 2024-11-08T13:28:51,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T13:28:51,643 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6ccfea20c990,39565,1731072524927] 2024-11-08T13:28:51,644 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6ccfea20c990,39565,1731072524927 already deleted, retry=false 2024-11-08T13:28:51,644 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 6ccfea20c990,39565,1731072524927 on 6ccfea20c990,40009,1731072524365 2024-11-08T13:28:51,650 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:28:51,654 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for 6ccfea20c990,39565,1731072524927 (carryingMeta=true) 6ccfea20c990,39565,1731072524927/CRASHED/regionCount=10/lock=java.util.concurrent.locks.ReentrantReadWriteLock@4f833239[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-08T13:28:51,654 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:28:51,655 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-08T13:28:51,655 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:28:51,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:28:51,655 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T13:28:51,718 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:28:51,720 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 0 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:28:51,721 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T13:28:51,721 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:28:51,721 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T13:28:51,721 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T13:28:51,722 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T13:28:51,722 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T13:28:51,722 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T13:28:51,722 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T13:28:51,723 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T13:28:51,727 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T13:28:51,728 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T13:28:51,729 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-08T13:28:51,729 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:28:51,729 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T13:28:51,730 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-08T13:28:51,731 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:28:51,731 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ccfea20c990,39565,1731072524927, table=hbase:meta, region=1588230740 2024-11-08T13:28:51,732 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ccfea20c990,39565,1731072524927, state=OPEN 2024-11-08T13:28:51,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:28:51,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:28:51,733 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:28:51,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:28:51,733 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:28:51,734 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:28:51,736 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731072561736 2024-11-08T13:28:51,736 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T13:28:51,736 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:28:51,736 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-08T13:28:51,738 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true, isMeta: true 2024-11-08T13:28:51,738 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:28:51,740 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting 2024-11-08T13:28:51,740 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:28:51,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:28:51,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:28:51,741 INFO [PEWorker-1 {}] master.SplitWALManager(105): 6ccfea20c990,39565,1731072524927 WAL count=1, meta=true 2024-11-08T13:28:51,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:28:51,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39565-0x1001f85d9a10001, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:28:51,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39565-0x1001f85d9a10001, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:28:51,744 INFO [RS:0;6ccfea20c990:39565 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T13:28:51,744 INFO [RS:0;6ccfea20c990:39565 {}] regionserver.HRegionServer(1031): Exiting; stopping=6ccfea20c990,39565,1731072524927; zookeeper connection closed. 2024-11-08T13:28:51,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta}] 2024-11-08T13:28:51,744 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-08T13:28:51,744 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:28:51,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:28:51,744 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4ec774d3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4ec774d3 2024-11-08T13:28:51,744 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T13:28:51,849 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:28:51,850 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:28:51,850 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6ccfea20c990:39565 this server is in the failed servers list 2024-11-08T13:28:51,850 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:28:51,850 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:28:51,850 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:28:52,059 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:28:52,060 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:28:52,060 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6ccfea20c990:39565 this server is in the failed servers list 2024-11-08T13:28:52,060 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:28:52,060 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:28:52,060 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:28:52,118 INFO [regionserver/6ccfea20c990:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-08T13:28:52,118 INFO [regionserver/6ccfea20c990:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-08T13:28:52,254 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T13:28:52,321 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true' 2024-11-08T13:28:52,325 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-08T13:28:52,369 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:28:52,369 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:28:52,370 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6ccfea20c990:39565 this server is in the failed servers list 2024-11-08T13:28:52,370 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:28:52,370 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:28:52,370 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:28:52,879 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:28:52,879 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:28:52,880 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6ccfea20c990:39565 this server is in the failed servers list 2024-11-08T13:28:52,880 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:28:52,880 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:28:52,880 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:28:53,889 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:28:53,890 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:28:53,891 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:28:53,897 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6ccfea20c990,39565,1731072524927 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2155 ms java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-08T13:28:53,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:28:53,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:28:53,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:28:53,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:28:54,698 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-08T13:28:54,699 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-08T13:28:54,701 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-08T13:28:54,701 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true Metrics about Tables on a single HBase RegionServer 2024-11-08T13:28:54,703 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T13:28:54,703 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-08T13:28:54,704 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-08T13:28:54,704 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-08T13:28:55,910 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:28:55,910 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:28:55,910 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:28:55,911 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6ccfea20c990,39565,1731072524927 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4174 ms java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-08T13:28:55,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:28:55,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:28:55,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:28:55,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:28:59,929 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:28:59,930 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:28:59,931 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:28:59,931 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6ccfea20c990,39565,1731072524927 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8194 ms java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-08T13:28:59,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:28:59,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:28:59,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:28:59,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:09,990 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:29:09,990 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:29:09,992 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:29:09,993 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6ccfea20c990,39565,1731072524927 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18256 ms java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-08T13:29:09,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:09,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:29:09,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:29:09,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:13,520 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T13:29:19,892 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=6ccfea20c990:40009 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-08T13:29:20,069 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:29:20,070 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:29:20,071 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:29:20,071 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6ccfea20c990,39565,1731072524927 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28334 ms java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-08T13:29:20,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:20,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:29:20,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:29:20,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:27,275 INFO [master/6ccfea20c990:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-08T13:29:27,276 INFO [master/6ccfea20c990:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-08T13:29:30,150 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:29:30,150 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:29:30,152 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:29:30,153 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6ccfea20c990,39565,1731072524927 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38416 ms java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-08T13:29:30,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:30,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:29:30,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:29:30,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:40,219 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:29:40,220 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:29:40,221 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:29:40,222 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6ccfea20c990,39565,1731072524927 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48485 ms java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-08T13:29:40,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:40,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:29:40,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:29:40,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:43,520 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T13:29:45,645 WARN [RS:2;6ccfea20c990:35367 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of 6ccfea20c990,39565,1731072524927: status=status unset, state=RUNNING, startTime=1731072531654, completionTime=-1 appears to have been leaked 2024-11-08T13:29:47,041 DEBUG [master/6ccfea20c990:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c6699738a1be7c8647568269e341753c changed from -1.0 to 0.0, refreshing cache 2024-11-08T13:29:47,042 DEBUG [master/6ccfea20c990:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0f0799092106200541bc0c416b0bdbb5 changed from -1.0 to 0.0, refreshing cache 2024-11-08T13:29:47,042 DEBUG [master/6ccfea20c990:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 286367177ffe2bf304003f3b829326ac changed from -1.0 to 0.0, refreshing cache 2024-11-08T13:29:47,043 DEBUG [master/6ccfea20c990:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e178bdc5865404fdcd9e707b6918569c changed from -1.0 to 0.0, refreshing cache 2024-11-08T13:29:47,043 DEBUG [master/6ccfea20c990:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5ab60b8cfddafd2fcce990e602b31aef changed from -1.0 to 0.0, refreshing cache 2024-11-08T13:29:47,043 DEBUG [master/6ccfea20c990:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c02aa10a67704d3f15deae347223dc13 changed from -1.0 to 0.0, refreshing cache 2024-11-08T13:29:47,044 DEBUG [master/6ccfea20c990:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 30f6256cfa21047086ecffea588e9713 changed from -1.0 to 0.0, refreshing cache 2024-11-08T13:29:47,044 DEBUG [master/6ccfea20c990:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4fbf6f08f931761acf639331e9fd9170 changed from -1.0 to 0.0, refreshing cache 2024-11-08T13:29:47,044 DEBUG [master/6ccfea20c990:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e1c2472f75aa6a0ed9ba5017889bd71b changed from -1.0 to 0.0, refreshing cache 2024-11-08T13:29:47,044 DEBUG [master/6ccfea20c990:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7e4837322188444dc48d5c8632c81ce5 changed from -1.0 to 0.0, refreshing cache 2024-11-08T13:29:47,044 DEBUG [master/6ccfea20c990:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d85397130af5b1fc184790336ef5d2b8 changed from -1.0 to 0.0, refreshing cache 2024-11-08T13:29:47,045 DEBUG [master/6ccfea20c990:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 50c49cba3ff42064b0e071b4b0d3656e changed from -1.0 to 0.0, refreshing cache 2024-11-08T13:29:47,045 DEBUG [master/6ccfea20c990:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5503a38fb5ee20ab8673683ba1a1bcaa changed from -1.0 to 0.0, refreshing cache 2024-11-08T13:29:47,045 DEBUG [master/6ccfea20c990:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0b082e035d4bc82dc2f93642d66c000c changed from -1.0 to 0.0, refreshing cache 2024-11-08T13:29:47,045 DEBUG [master/6ccfea20c990:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 223bf549b03e7990028b7927dd897a23 changed from -1.0 to 0.0, refreshing cache 2024-11-08T13:29:47,045 DEBUG [master/6ccfea20c990:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b7f58040b9958dcdcc78f264a8778203 changed from -1.0 to 0.0, refreshing cache 2024-11-08T13:29:47,046 DEBUG [master/6ccfea20c990:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f973a563f81a494eeb1443f81f0bb069 changed from -1.0 to 0.0, refreshing cache 2024-11-08T13:29:49,552 INFO [regionserver/6ccfea20c990:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-08T13:29:49,895 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=6ccfea20c990:40009 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-08T13:29:51,742 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-08T13:28:51.742Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:28:51.850Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:28:52.060Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:28:52.370Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:28:52.880Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:28:53.899Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:28:55.911Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:28:59.932Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:09.994Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:20.072Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:30.155Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:40.223Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T13:29:51,746 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 1 exec state=pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:29:51,746 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T13:29:51,746 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:29:51,746 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T13:29:51,746 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T13:29:51,747 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T13:29:51,747 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T13:29:51,747 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T13:29:51,748 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T13:29:51,750 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T13:29:51,754 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T13:29:51,754 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T13:29:51,755 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-08T13:29:51,755 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=false; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:29:51,755 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=false; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T13:29:51,755 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-08T13:29:51,756 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:29:51,756 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ccfea20c990,39565,1731072524927, table=hbase:meta, region=1588230740 2024-11-08T13:29:51,756 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ccfea20c990,39565,1731072524927, state=OPEN 2024-11-08T13:29:51,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:29:51,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:29:51,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:29:51,757 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:29:51,757 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:29:51,757 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:29:51,758 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731072621758 2024-11-08T13:29:51,758 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T13:29:51,758 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true, isMeta: true 2024-11-08T13:29:51,758 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:29:51,759 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:29:51,759 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:29:51,760 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:51,760 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:29:51,760 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:29:51,760 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:51,760 INFO [PEWorker-1 {}] master.SplitWALManager(105): 6ccfea20c990,39565,1731072524927 WAL count=1, meta=true 2024-11-08T13:29:51,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta}] 2024-11-08T13:29:51,761 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-08T13:29:51,766 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=6ccfea20c990,35367,1731072525051 2024-11-08T13:29:51,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051}] 2024-11-08T13:29:51,870 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:29:51,870 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:29:51,871 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6ccfea20c990:39565 this server is in the failed servers list 2024-11-08T13:29:51,871 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:29:51,872 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:29:51,872 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:29:51,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35367 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-08T13:29:51,963 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, size=52.9 K (54214bytes) 2024-11-08T13:29:51,963 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-08T13:29:51,964 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-08T13:29:51,964 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta 2024-11-08T13:29:51,968 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta after 3ms 2024-11-08T13:29:51,975 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta took 14ms 2024-11-08T13:29:51,983 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 2024-11-08T13:29:51,990 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta.temp 2024-11-08T13:29:51,993 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta.temp 2024-11-08T13:29:51,994 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta so closing down 2024-11-08T13:29:51,995 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-08T13:29:51,998 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-08T13:29:52,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741864_1040 (size=53973) 2024-11-08T13:29:52,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741864_1040 (size=53973) 2024-11-08T13:29:52,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741864_1040 (size=53973) 2024-11-08T13:29:52,010 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) 2024-11-08T13:29:52,012 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta.temp to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-08T13:29:52,017 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, size=52.9 K, length=54214, corrupted=false, cancelled=false 2024-11-08T13:29:52,017 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, journal: Splitting hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, size=52.9 K (54214bytes) at 1731072591963Creating recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta.temp at 1731072591993 (+30 ms)Finishing writing output for hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta so closing down at 1731072591994 (+1 ms)3 split writer threads finished at 1731072591998 (+4 ms)Closed recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) at 1731072592010 (+12 ms)Rename recovered edits hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta.temp to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1731072592012 (+2 ms)Processed 58 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, size=52.9 K, length=54214, corrupted=false, cancelled=false at 1731072592017 (+5 ms) 2024-11-08T13:29:52,018 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta 2024-11-08T13:29:52,019 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-08T13:29:52,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40009 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-08T13:29:52,029 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/oldWALs 2024-11-08T13:29:52,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-08T13:29:52,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 in 261 msec 2024-11-08T13:29:52,034 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=6ccfea20c990,35367,1731072525051 2024-11-08T13:29:52,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-08T13:29:52,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 in 273 msec 2024-11-08T13:29:52,040 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting dir is empty, no logs to split. 2024-11-08T13:29:52,040 INFO [PEWorker-1 {}] master.SplitWALManager(105): 6ccfea20c990,39565,1731072524927 WAL count=0, meta=true 2024-11-08T13:29:52,040 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 6ccfea20c990,39565,1731072524927 WAL splitting is done? wals=0, meta=true 2024-11-08T13:29:52,040 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-08T13:29:52,040 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:29:52,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:29:52,040 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T13:29:52,080 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:29:52,080 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:29:52,080 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6ccfea20c990:39565 this server is in the failed servers list 2024-11-08T13:29:52,081 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:29:52,081 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:29:52,081 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:29:52,390 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:29:52,390 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:29:52,391 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6ccfea20c990:39565 this server is in the failed servers list 2024-11-08T13:29:52,391 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:29:52,391 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:29:52,391 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:29:52,900 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:29:52,900 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:29:52,901 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6ccfea20c990:39565 this server is in the failed servers list 2024-11-08T13:29:52,902 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:29:52,902 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:29:52,902 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:29:53,909 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:29:53,910 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:29:53,912 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:29:53,913 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6ccfea20c990,39565,1731072524927 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2154 ms java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-08T13:29:53,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:53,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:29:53,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:29:53,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:55,929 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:29:55,930 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:29:55,931 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:29:55,932 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6ccfea20c990,39565,1731072524927 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4173 ms java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-08T13:29:55,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:55,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:29:55,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:29:55,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:59,969 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:29:59,969 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:29:59,971 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:29:59,972 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6ccfea20c990,39565,1731072524927 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8213 ms java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-08T13:29:59,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:59,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:29:59,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:29:59,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:10,049 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:30:10,050 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:30:10,052 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:30:10,053 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6ccfea20c990,39565,1731072524927 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18294 ms java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-08T13:30:10,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:10,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:30:10,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:30:10,055 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:13,520 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T13:30:20,120 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:30:20,120 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:30:20,122 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:30:20,124 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6ccfea20c990,39565,1731072524927 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28365 ms java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-08T13:30:20,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:20,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:30:20,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:30:20,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:30,190 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:30:30,190 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:30:30,192 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:30:30,193 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6ccfea20c990,39565,1731072524927 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38434 ms java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-08T13:30:30,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:30,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:30:30,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:30:30,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:40,240 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:30:40,240 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:30:40,242 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:30:40,243 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6ccfea20c990,39565,1731072524927 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48484 ms java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-08T13:30:40,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:40,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:30:40,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:30:40,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:43,521 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T13:30:46,111 INFO [regionserver/6ccfea20c990:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactedHFilesCleaner was stopped 2024-11-08T13:30:46,141 WARN [regionserver/6ccfea20c990:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 23, running: 1 2024-11-08T13:30:46,213 WARN [regionserver/6ccfea20c990:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 21, running: 1 2024-11-08T13:30:51,760 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-08T13:29:51.760Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:51.872Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:29:52.081Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:29:52.391Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:29:52.902Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:29:53.915Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:55.934Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:29:59.974Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:10.055Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:20.125Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:30.195Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:40.245Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T13:30:51,762 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 2 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:30:51,763 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T13:30:51,763 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:30:51,763 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T13:30:51,763 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T13:30:51,763 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T13:30:51,764 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T13:30:51,764 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T13:30:51,764 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T13:30:51,766 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T13:30:51,770 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T13:30:51,770 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T13:30:51,770 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T13:30:51,770 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T13:30:51,771 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-08T13:30:51,772 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=false; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:30:51,772 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 2024-11-08T13:30:51,772 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 2024-11-08T13:30:51,772 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=false; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T13:30:51,772 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:30:51,772 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:30:51,772 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-08T13:30:51,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:30:51,773 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ccfea20c990,39565,1731072524927, table=hbase:meta, region=1588230740 2024-11-08T13:30:51,773 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ccfea20c990,39565,1731072524927, state=OPEN 2024-11-08T13:30:51,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:30:51,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:30:51,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:30:51,775 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:30:51,775 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:30:51,775 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:30:51,775 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731072681775 2024-11-08T13:30:51,775 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T13:30:51,776 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:30:51,776 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:30:51,777 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:30:51,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:51,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:30:51,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:30:51,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:51,777 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting dir is empty, no logs to split. 2024-11-08T13:30:51,777 INFO [PEWorker-1 {}] master.SplitWALManager(105): 6ccfea20c990,39565,1731072524927 WAL count=0, meta=true 2024-11-08T13:30:51,777 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 6ccfea20c990,39565,1731072524927 WAL splitting is done? wals=0, meta=true 2024-11-08T13:30:51,777 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-08T13:30:51,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T13:30:51,779 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-08T13:30:51,779 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:30:51,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:30:51,780 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T13:30:51,880 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:30:51,880 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:30:51,881 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6ccfea20c990:39565 this server is in the failed servers list 2024-11-08T13:30:51,881 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:30:51,881 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:30:51,881 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:30:52,090 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:30:52,090 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:30:52,091 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6ccfea20c990:39565 this server is in the failed servers list 2024-11-08T13:30:52,091 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:30:52,092 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:30:52,092 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:30:52,400 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:30:52,400 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:30:52,400 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6ccfea20c990:39565 this server is in the failed servers list 2024-11-08T13:30:52,401 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:30:52,401 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:30:52,401 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:30:52,910 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:30:52,910 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:30:52,911 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6ccfea20c990:39565 this server is in the failed servers list 2024-11-08T13:30:52,911 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:30:52,912 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:30:52,912 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:30:53,929 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:30:53,930 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:30:53,931 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:30:53,932 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6ccfea20c990,39565,1731072524927 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2156 ms java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-08T13:30:53,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:53,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:30:53,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:30:53,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:55,960 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:30:55,960 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:30:55,961 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:30:55,962 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6ccfea20c990,39565,1731072524927 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4186 ms java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-08T13:30:55,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:55,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:30:55,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:30:55,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:59,970 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:30:59,970 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:30:59,972 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:30:59,973 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6ccfea20c990,39565,1731072524927 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8197 ms java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-08T13:30:59,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:59,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:30:59,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:30:59,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:31:00,897 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=6ccfea20c990:40009 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-08T13:31:10,029 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:31:10,030 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:31:10,031 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:31:10,032 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6ccfea20c990,39565,1731072524927 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18256 ms java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-08T13:31:10,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:31:10,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:31:10,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:31:10,035 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:31:13,521 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T13:31:20,040 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:31:20,040 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:31:20,043 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:31:20,044 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6ccfea20c990,39565,1731072524927 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28267 ms java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-08T13:31:20,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:31:20,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:31:20,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:31:20,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:31:30,069 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:31:30,069 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:31:30,070 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:31:30,071 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6ccfea20c990,39565,1731072524927 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38294 ms java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-08T13:31:30,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:31:30,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:31:30,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:31:30,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:31:40,160 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:31:40,160 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:31:40,162 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:31:40,164 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 6ccfea20c990,39565,1731072524927 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48387 ms java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-08T13:31:40,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:31:40,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:31:40,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:31:40,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:31:43,522 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T13:31:51,781 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-08T13:30:51.777Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:51.881Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:30:52.092Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:30:52.401Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:30:52.912Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:30:53.934Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:55.963Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:30:59.975Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:31:10.034Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:31:20.045Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:31:30.071Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:31:40.165Z, java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-08T13:31:51,784 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 3 exec state=pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:31:51,785 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T13:31:51,785 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:31:51,785 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T13:31:51,785 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T13:31:51,785 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T13:31:51,785 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T13:31:51,785 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T13:31:51,786 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T13:31:51,787 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T13:31:51,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T13:31:51,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T13:31:51,790 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T13:31:51,790 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T13:31:51,791 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-08T13:31:51,791 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=false; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:31:51,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:51,792 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:51,792 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=false; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T13:31:51,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:51,792 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:51,792 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-08T13:31:51,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:51,793 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ccfea20c990,39565,1731072524927, table=hbase:meta, region=1588230740 2024-11-08T13:31:51,793 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ccfea20c990,39565,1731072524927, state=OPEN 2024-11-08T13:31:51,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:51,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:51,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:51,795 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:51,795 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:51,795 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:51,796 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731072741796 2024-11-08T13:31:51,796 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T13:31:51,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T13:31:51,797 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-08T13:31:51,797 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:31:51,797 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:31:51,798 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:31:51,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:31:51,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:31:51,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:31:51,799 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:31:51,799 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T13:31:51,800 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T13:31:51,910 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:31:51,910 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1] 2024-11-08T13:31:51,910 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6ccfea20c990:39565 this server is in the failed servers list 2024-11-08T13:31:51,910 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:31:51,910 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:31:51,910 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:31:51,950 DEBUG [6ccfea20c990:40009 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-08T13:31:51,951 DEBUG [6ccfea20c990:40009 {}] balancer.BalancerClusterState(204): Hosts are {6ccfea20c990=0} racks are {/default-rack=0} 2024-11-08T13:31:51,951 DEBUG [6ccfea20c990:40009 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T13:31:51,951 DEBUG [6ccfea20c990:40009 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T13:31:51,951 DEBUG [6ccfea20c990:40009 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T13:31:51,951 DEBUG [6ccfea20c990:40009 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T13:31:51,951 INFO [6ccfea20c990:40009 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T13:31:51,951 INFO [6ccfea20c990:40009 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T13:31:51,951 DEBUG [6ccfea20c990:40009 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T13:31:51,951 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:51,953 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ccfea20c990,35367,1731072525051, state=OPENING 2024-11-08T13:31:51,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:51,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:51,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:51,954 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T13:31:51,954 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:51,954 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:51,954 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:51,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6ccfea20c990,35367,1731072525051}] 2024-11-08T13:31:52,111 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42957, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T13:31:52,119 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T13:31:52,119 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:31:52,119 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T13:31:52,120 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,35367,1731072525051, seqNum=-1] 2024-11-08T13:31:52,120 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T13:31:52,120 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T13:31:52,122 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44829, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T13:31:52,124 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6ccfea20c990%2C35367%2C1731072525051.meta, suffix=.meta, logDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,35367,1731072525051, archiveDir=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/oldWALs, maxLogs=32 2024-11-08T13:31:52,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35367 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Scan size: 121 connection: 172.17.0.2:44829 deadline: 1731072771799, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 6ccfea20c990,35367,1731072525051 2024-11-08T13:31:52,137 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,35367,1731072525051/6ccfea20c990%2C35367%2C1731072525051.meta.1731072712125.meta, exclude list is [], retry=0 2024-11-08T13:31:52,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36805,DS-067499be-1a0c-4eb6-83cf-12ed63a0b51c,DISK] 2024-11-08T13:31:52,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38513,DS-4c23f8af-3e26-4caf-af50-9edfbeedfc0d,DISK] 2024-11-08T13:31:52,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,35367,1731072525051, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,35367,1731072525051, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 6ccfea20c990,35367,1731072525051 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T13:31:52,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,35367,1731072525051, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 6ccfea20c990,35367,1731072525051 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T13:31:52,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,35367,1731072525051, seqNum=-1 from cache 2024-11-08T13:31:52,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36145,DS-d3e6e3df-bbf9-4750-9cd5-a9cb825eda4a,DISK] 2024-11-08T13:31:52,148 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,35367,1731072525051/6ccfea20c990%2C35367%2C1731072525051.meta.1731072712125.meta 2024-11-08T13:31:52,148 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41369:41369),(127.0.0.1/127.0.0.1:44267:44267),(127.0.0.1/127.0.0.1:43697:43697)] 2024-11-08T13:31:52,148 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:31:52,149 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T13:31:52,149 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T13:31:52,149 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T13:31:52,149 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T13:31:52,149 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:31:52,149 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T13:31:52,150 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T13:31:52,151 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T13:31:52,152 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T13:31:52,152 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:52,153 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:31:52,153 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T13:31:52,154 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T13:31:52,154 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:52,154 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:31:52,154 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T13:31:52,155 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T13:31:52,155 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:52,156 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:31:52,156 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T13:31:52,157 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T13:31:52,157 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:52,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:31:52,157 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T13:31:52,158 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740 2024-11-08T13:31:52,160 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740 2024-11-08T13:31:52,161 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-08T13:31:52,174 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-08T13:31:52,190 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=49.58 KB heapSize=69.98 KB 2024-11-08T13:31:52,223 WARN [RS:2;6ccfea20c990:35367 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of 6ccfea20c990,39565,1731072524927: status=Processing ServerCrashProcedure of 6ccfea20c990,39565,1731072524927 current State SERVER_CRASH_SPLIT_META_LOGS, state=RUNNING, startTime=1731072531729, completionTime=-1 appears to have been leaked 2024-11-08T13:31:52,223 WARN [RS:2;6ccfea20c990:35367 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of 6ccfea20c990,39565,1731072524927: status=Processing ServerCrashProcedure of 6ccfea20c990,39565,1731072524927 current State SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, state=RUNNING, startTime=1731072591755, completionTime=-1 appears to have been leaked 2024-11-08T13:31:52,223 WARN [RS:2;6ccfea20c990:35367 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of 6ccfea20c990,39565,1731072524927: status=Processing ServerCrashProcedure of 6ccfea20c990,39565,1731072524927 current State SERVER_CRASH_ASSIGN_META, state=RUNNING, startTime=1731072651772, completionTime=-1 appears to have been leaked 2024-11-08T13:31:52,270 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/.tmp/info/d432f83c3bc64160b5365977218127c2 is 268, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018./info:regioninfo/1731072527931/Put/seqid=0 2024-11-08T13:31:52,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741866_1042 (size=58053) 2024-11-08T13:31:52,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741866_1042 (size=58053) 2024-11-08T13:31:52,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741866_1042 (size=58053) 2024-11-08T13:31:52,285 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.31 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/.tmp/info/d432f83c3bc64160b5365977218127c2 2024-11-08T13:31:52,346 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/.tmp/ns/1b83672ab78d419689d4a1a3b5b0656b is 43, key is default/ns:d/1731072526860/Put/seqid=0 2024-11-08T13:31:52,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741867_1043 (size=5153) 2024-11-08T13:31:52,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741867_1043 (size=5153) 2024-11-08T13:31:52,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741867_1043 (size=5153) 2024-11-08T13:31:52,450 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:31:52,450 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,35367,1731072525051, seqNum=-1] 2024-11-08T13:31:52,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35367 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 121 connection: 172.17.0.2:44829 deadline: 1731072771797, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 6ccfea20c990,35367,1731072525051 2024-11-08T13:31:52,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,35367,1731072525051, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,35367,1731072525051, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 6ccfea20c990,35367,1731072525051 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T13:31:52,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,35367,1731072525051, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 6ccfea20c990,35367,1731072525051 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T13:31:52,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,35367,1731072525051, seqNum=-1 from cache 2024-11-08T13:31:52,755 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/.tmp/ns/1b83672ab78d419689d4a1a3b5b0656b 2024-11-08T13:31:52,782 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/.tmp/table/aad80632fa7447b3807407454fb7d5d9 is 105, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/table:state/1731072528406/Put/seqid=0 2024-11-08T13:31:52,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741868_1044 (size=5620) 2024-11-08T13:31:52,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741868_1044 (size=5620) 2024-11-08T13:31:52,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741868_1044 (size=5620) 2024-11-08T13:31:52,790 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=202 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/.tmp/table/aad80632fa7447b3807407454fb7d5d9 2024-11-08T13:31:52,798 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/.tmp/info/d432f83c3bc64160b5365977218127c2 as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/info/d432f83c3bc64160b5365977218127c2 2024-11-08T13:31:52,805 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/info/d432f83c3bc64160b5365977218127c2, entries=260, sequenceid=60, filesize=56.7 K 2024-11-08T13:31:52,808 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/.tmp/ns/1b83672ab78d419689d4a1a3b5b0656b as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/ns/1b83672ab78d419689d4a1a3b5b0656b 2024-11-08T13:31:52,815 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/ns/1b83672ab78d419689d4a1a3b5b0656b, entries=2, sequenceid=60, filesize=5.0 K 2024-11-08T13:31:52,816 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/.tmp/table/aad80632fa7447b3807407454fb7d5d9 as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/table/aad80632fa7447b3807407454fb7d5d9 2024-11-08T13:31:52,823 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/table/aad80632fa7447b3807407454fb7d5d9, entries=2, sequenceid=60, filesize=5.5 K 2024-11-08T13:31:52,828 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~49.58 KB/50772, heapSize ~69.69 KB/71360, currentSize=0 B/0 for 1588230740 in 650ms, sequenceid=60, compaction requested=false; wal=null 2024-11-08T13:31:52,829 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-08T13:31:52,831 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T13:31:52,831 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T13:31:52,832 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T13:31:52,833 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T13:31:52,836 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-08T13:31:52,837 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59897566, jitterRate=-0.10745671391487122}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T13:31:52,837 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T13:31:52,837 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731072712150Writing region info on filesystem at 1731072712150Initializing all the Stores at 1731072712151 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072712151Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072712151Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072712151Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072712151Obtaining lock to block concurrent updates at 1731072712190 (+39 ms)Preparing flush snapshotting stores in 1588230740 at 1731072712190Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=50772, getHeapSize=71600, getOffHeapSize=0, getCellsCount=264 at 1731072712193 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731072712193Flushing 1588230740/info: creating writer at 1731072712197 (+4 ms)Flushing 1588230740/info: appending metadata at 1731072712258 (+61 ms)Flushing 1588230740/info: closing flushed file at 1731072712261 (+3 ms)Flushing 1588230740/ns: creating writer at 1731072712325 (+64 ms)Flushing 1588230740/ns: appending metadata at 1731072712344 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1731072712344Flushing 1588230740/table: creating writer at 1731072712765 (+421 ms)Flushing 1588230740/table: appending metadata at 1731072712780 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731072712780Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ce200ff: reopening flushed file at 1731072712797 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6da53c83: reopening flushed file at 1731072712807 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bc24: reopening flushed file at 1731072712815 (+8 ms)Finished flush of dataSize ~49.58 KB/50772, heapSize ~69.69 KB/71360, currentSize=0 B/0 for 1588230740 in 650ms, sequenceid=60, compaction requested=false; wal=null at 1731072712828 (+13 ms)Cleaning up temporary data from old regions at 1731072712831 (+3 ms)Running coprocessor post-open hooks at 1731072712837 (+6 ms)Region opened successfully at 1731072712837 2024-11-08T13:31:52,838 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1731072712107 2024-11-08T13:31:52,840 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T13:31:52,840 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T13:31:52,841 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:52,842 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ccfea20c990,35367,1731072525051, state=OPEN 2024-11-08T13:31:52,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:52,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:52,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:52,843 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:52,843 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:52,843 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:52,844 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:52,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-08T13:31:52,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ccfea20c990,35367,1731072525051 in 890 msec 2024-11-08T13:31:52,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-08T13:31:52,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0510 sec 2024-11-08T13:31:52,959 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:31:52,960 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,35367,1731072525051, seqNum=-1] 2024-11-08T13:31:52,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6699738a1be7c8647568269e341753c, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:52,996 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d5935497231430a7057c66d28a9d018, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:52,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7e4837322188444dc48d5c8632c81ce5, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:52,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8fbd70358d0bf3e3f164569432ecfe, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:52,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f0799092106200541bc0c416b0bdbb5, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:52,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d85397130af5b1fc184790336ef5d2b8, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:52,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e1216db90b6f396355f2b6bc308ffe, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:52,997 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=286367177ffe2bf304003f3b829326ac, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:52,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50c49cba3ff42064b0e071b4b0d3656e, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:52,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5970d249471986ac22ae194d250f9a38, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:52,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5503a38fb5ee20ab8673683ba1a1bcaa, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:52,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e178bdc5865404fdcd9e707b6918569c, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:52,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ef0046eead8230ad7213c161e3532bd, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:52,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b082e035d4bc82dc2f93642d66c000c, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:52,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ab60b8cfddafd2fcce990e602b31aef, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:52,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06547784f6904a15d1d127f091988ed3, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:52,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c02aa10a67704d3f15deae347223dc13, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:52,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcf86fec97b560cab0f86ca4727e4e57, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:52,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=223bf549b03e7990028b7927dd897a23, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:52,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30f6256cfa21047086ecffea588e9713, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:52,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7f58040b9958dcdcc78f264a8778203, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:52,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00f5b98ab61f3475c8968b8a7a9f6cb4, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:52,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fbf6f08f931761acf639331e9fd9170, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:52,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f973a563f81a494eeb1443f81f0bb069, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:52,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99abbd965e76aa0c5fe5a1011e45a9ae, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:52,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1c2472f75aa6a0ed9ba5017889bd71b, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:53,000 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-08T13:31:53,000 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731072773000 2024-11-08T13:31:53,000 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731072833000 2024-11-08T13:31:53,000 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 1.2030 sec 2024-11-08T13:31:53,000 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 6ccfea20c990,39565,1731072524927 had 10 regions 2024-11-08T13:31:53,000 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-08T13:31:53,001 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:31:53,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:31:53,001 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T13:31:53,250 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 4 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:31:53,250 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T13:31:53,250 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:31:53,250 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T13:31:53,250 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T13:31:53,251 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T13:31:53,251 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T13:31:53,251 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T13:31:53,251 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T13:31:53,252 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T13:31:53,254 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40009 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T13:31:53,254 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40009 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T13:31:53,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T13:31:53,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T13:31:53,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T13:31:53,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T13:31:53,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T13:31:53,256 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T13:31:53,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-08T13:31:53,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, hasLock=false; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:31:53,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:53,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:53,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T13:31:53,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:53,258 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, hasLock=false; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T13:31:53,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:53,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:53,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:53,258 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:53,259 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-08T13:31:53,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=61 2024-11-08T13:31:53,260 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ccfea20c990,35367,1731072525051, table=hbase:meta, region=1588230740 2024-11-08T13:31:53,260 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ccfea20c990,35367,1731072525051, state=OPEN 2024-11-08T13:31:53,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:53,262 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:53,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:53,262 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:53,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:53,262 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:53,265 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731072743265 2024-11-08T13:31:53,266 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T13:31:53,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6699738a1be7c8647568269e341753c, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:53,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d5935497231430a7057c66d28a9d018, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:53,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7e4837322188444dc48d5c8632c81ce5, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:53,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8fbd70358d0bf3e3f164569432ecfe, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:53,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f0799092106200541bc0c416b0bdbb5, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:53,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d85397130af5b1fc184790336ef5d2b8, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:53,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e1216db90b6f396355f2b6bc308ffe, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:53,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=286367177ffe2bf304003f3b829326ac, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:53,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50c49cba3ff42064b0e071b4b0d3656e, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:53,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5970d249471986ac22ae194d250f9a38, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:53,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5503a38fb5ee20ab8673683ba1a1bcaa, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:53,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e178bdc5865404fdcd9e707b6918569c, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:53,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ef0046eead8230ad7213c161e3532bd, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:53,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b082e035d4bc82dc2f93642d66c000c, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:53,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ab60b8cfddafd2fcce990e602b31aef, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:53,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06547784f6904a15d1d127f091988ed3, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:53,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c02aa10a67704d3f15deae347223dc13, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:53,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcf86fec97b560cab0f86ca4727e4e57, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:53,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=223bf549b03e7990028b7927dd897a23, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:53,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30f6256cfa21047086ecffea588e9713, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:53,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7f58040b9958dcdcc78f264a8778203, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:53,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00f5b98ab61f3475c8968b8a7a9f6cb4, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:53,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fbf6f08f931761acf639331e9fd9170, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:53,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f973a563f81a494eeb1443f81f0bb069, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:53,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99abbd965e76aa0c5fe5a1011e45a9ae, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:53,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1c2472f75aa6a0ed9ba5017889bd71b, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:53,279 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-08T13:31:53,279 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731072773279 2024-11-08T13:31:53,279 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731072833279 2024-11-08T13:31:53,279 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-08T13:31:53,279 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 6ccfea20c990,39565,1731072524927 had 9 regions 2024-11-08T13:31:53,279 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-08T13:31:53,281 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true, isMeta: false 2024-11-08T13:31:53,283 INFO [PEWorker-1 {}] master.SplitWALManager(105): 6ccfea20c990,39565,1731072524927 WAL count=1, meta=false 2024-11-08T13:31:53,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.1731072526271}] 2024-11-08T13:31:53,283 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-08T13:31:53,284 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:31:53,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:31:53,284 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T13:31:53,529 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 5 exec state=pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:31:53,529 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T13:31:53,529 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:31:53,529 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T13:31:53,529 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T13:31:53,530 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T13:31:53,530 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T13:31:53,530 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T13:31:53,530 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T13:31:53,531 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T13:31:53,532 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T13:31:53,532 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T13:31:53,532 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T13:31:53,532 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T13:31:53,532 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T13:31:53,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T13:31:53,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-08T13:31:53,534 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=false; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:31:53,534 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:53,534 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:53,534 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T13:31:53,534 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:53,534 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=false; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T13:31:53,534 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:53,534 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:53,534 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:53,534 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:53,534 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-08T13:31:53,535 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=61 2024-11-08T13:31:53,535 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ccfea20c990,35367,1731072525051, table=hbase:meta, region=1588230740 2024-11-08T13:31:53,535 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ccfea20c990,35367,1731072525051, state=OPEN 2024-11-08T13:31:53,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:53,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:53,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:53,536 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:53,536 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:53,536 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:53,539 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731072743539 2024-11-08T13:31:53,539 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T13:31:53,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6699738a1be7c8647568269e341753c, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:53,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d5935497231430a7057c66d28a9d018, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:53,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7e4837322188444dc48d5c8632c81ce5, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:53,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8fbd70358d0bf3e3f164569432ecfe, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:53,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f0799092106200541bc0c416b0bdbb5, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:53,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d85397130af5b1fc184790336ef5d2b8, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:53,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e1216db90b6f396355f2b6bc308ffe, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:53,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=286367177ffe2bf304003f3b829326ac, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:53,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50c49cba3ff42064b0e071b4b0d3656e, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:53,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5970d249471986ac22ae194d250f9a38, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:53,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5503a38fb5ee20ab8673683ba1a1bcaa, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:53,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e178bdc5865404fdcd9e707b6918569c, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:53,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ef0046eead8230ad7213c161e3532bd, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:53,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b082e035d4bc82dc2f93642d66c000c, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:53,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ab60b8cfddafd2fcce990e602b31aef, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:53,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06547784f6904a15d1d127f091988ed3, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:53,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c02aa10a67704d3f15deae347223dc13, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:53,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcf86fec97b560cab0f86ca4727e4e57, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:53,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=223bf549b03e7990028b7927dd897a23, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:53,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30f6256cfa21047086ecffea588e9713, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:53,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7f58040b9958dcdcc78f264a8778203, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:53,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00f5b98ab61f3475c8968b8a7a9f6cb4, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:53,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fbf6f08f931761acf639331e9fd9170, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:53,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f973a563f81a494eeb1443f81f0bb069, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:53,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99abbd965e76aa0c5fe5a1011e45a9ae, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:53,548 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1c2472f75aa6a0ed9ba5017889bd71b, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:53,548 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-08T13:31:53,548 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731072773548 2024-11-08T13:31:53,548 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731072833548 2024-11-08T13:31:53,548 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-08T13:31:53,548 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true, isMeta: false 2024-11-08T13:31:53,551 INFO [PEWorker-1 {}] master.SplitWALManager(105): 6ccfea20c990,39565,1731072524927 WAL count=1, meta=false 2024-11-08T13:31:53,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.1731072526271}] 2024-11-08T13:31:53,551 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-08T13:31:53,553 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:53,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.1731072526271, worker=6ccfea20c990,35061,1731072525012}] 2024-11-08T13:31:53,708 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33295, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T13:31:53,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35061 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-08T13:31:53,723 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.1731072526271, size=327.1 K (334920bytes) 2024-11-08T13:31:53,724 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.1731072526271 2024-11-08T13:31:53,724 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.1731072526271 after 0ms 2024-11-08T13:31:53,728 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.1731072526271 took 5ms 2024-11-08T13:31:53,736 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 5970d249471986ac22ae194d250f9a38: last_flushed_sequence_id: 18446744073709551615 2024-11-08T13:31:53,737 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 4d5935497231430a7057c66d28a9d018: last_flushed_sequence_id: 18446744073709551615 2024-11-08T13:31:53,738 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 99abbd965e76aa0c5fe5a1011e45a9ae: last_flushed_sequence_id: 18446744073709551615 2024-11-08T13:31:53,739 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 00f5b98ab61f3475c8968b8a7a9f6cb4: last_flushed_sequence_id: 18446744073709551615 2024-11-08T13:31:53,740 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for c4e1216db90b6f396355f2b6bc308ffe: last_flushed_sequence_id: 18446744073709551615 2024-11-08T13:31:53,741 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 06547784f6904a15d1d127f091988ed3: last_flushed_sequence_id: 18446744073709551615 2024-11-08T13:31:53,743 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 5ef0046eead8230ad7213c161e3532bd: last_flushed_sequence_id: 18446744073709551615 2024-11-08T13:31:53,744 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 7c8fbd70358d0bf3e3f164569432ecfe: last_flushed_sequence_id: 18446744073709551615 2024-11-08T13:31:53,745 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for fcf86fec97b560cab0f86ca4727e4e57: last_flushed_sequence_id: 18446744073709551615 2024-11-08T13:31:53,753 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp 2024-11-08T13:31:53,756 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp 2024-11-08T13:31:53,756 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp 2024-11-08T13:31:53,760 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp 2024-11-08T13:31:53,761 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp 2024-11-08T13:31:53,763 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp 2024-11-08T13:31:53,764 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp 2024-11-08T13:31:53,766 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp 2024-11-08T13:31:53,767 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.1731072526271 so closing down 2024-11-08T13:31:53,767 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-08T13:31:53,768 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp 2024-11-08T13:31:53,768 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp 2024-11-08T13:31:53,770 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp 2024-11-08T13:31:53,772 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp 2024-11-08T13:31:53,772 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp 2024-11-08T13:31:53,773 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp 2024-11-08T13:31:53,774 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp 2024-11-08T13:31:53,774 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp 2024-11-08T13:31:53,776 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp 2024-11-08T13:31:53,781 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp 2024-11-08T13:31:53,782 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-08T13:31:53,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741869_1045 (size=36767) 2024-11-08T13:31:53,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741869_1045 (size=36767) 2024-11-08T13:31:53,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741869_1045 (size=36767) 2024-11-08T13:31:53,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741871_1047 (size=36767) 2024-11-08T13:31:53,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741871_1047 (size=36767) 2024-11-08T13:31:53,801 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-08T13:31:53,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741871_1047 (size=36767) 2024-11-08T13:31:53,803 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-08T13:31:53,803 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3/recovered.edits/0000000000000000004 2024-11-08T13:31:53,805 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57/recovered.edits/0000000000000000004 2024-11-08T13:31:53,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741870_1046 (size=36767) 2024-11-08T13:31:53,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741870_1046 (size=36767) 2024-11-08T13:31:53,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741870_1046 (size=36767) 2024-11-08T13:31:53,812 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-08T13:31:53,814 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018/recovered.edits/0000000000000000004 2024-11-08T13:31:53,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741873_1049 (size=36767) 2024-11-08T13:31:53,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741873_1049 (size=36767) 2024-11-08T13:31:53,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741873_1049 (size=36767) 2024-11-08T13:31:53,823 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-08T13:31:53,825 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd/recovered.edits/0000000000000000004 2024-11-08T13:31:53,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741872_1048 (size=36767) 2024-11-08T13:31:53,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741872_1048 (size=36767) 2024-11-08T13:31:53,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741872_1048 (size=36767) 2024-11-08T13:31:53,833 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-08T13:31:53,835 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4/recovered.edits/0000000000000000004 2024-11-08T13:31:53,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741874_1050 (size=36767) 2024-11-08T13:31:53,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741874_1050 (size=36767) 2024-11-08T13:31:53,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741874_1050 (size=36767) 2024-11-08T13:31:53,837 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-08T13:31:53,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741876_1052 (size=36767) 2024-11-08T13:31:53,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741876_1052 (size=36767) 2024-11-08T13:31:53,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741876_1052 (size=36767) 2024-11-08T13:31:53,843 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-08T13:31:53,843 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae/recovered.edits/0000000000000000004 2024-11-08T13:31:53,844 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe/recovered.edits/0000000000000000004 2024-11-08T13:31:53,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741875_1051 (size=36767) 2024-11-08T13:31:53,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741875_1051 (size=36767) 2024-11-08T13:31:53,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741875_1051 (size=36767) 2024-11-08T13:31:53,847 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-08T13:31:53,849 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38/recovered.edits/0000000000000000004 2024-11-08T13:31:53,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741877_1053 (size=36767) 2024-11-08T13:31:53,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741877_1053 (size=36767) 2024-11-08T13:31:53,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741877_1053 (size=36767) 2024-11-08T13:31:53,856 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-08T13:31:53,858 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe/recovered.edits/0000000000000000004 2024-11-08T13:31:53,858 INFO [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 18 edits across 9 Regions in 125 ms; skipped=9; WAL=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.1731072526271, size=327.1 K, length=334920, corrupted=false, cancelled=false 2024-11-08T13:31:53,858 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.1731072526271, journal: Splitting hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.1731072526271, size=327.1 K (334920bytes) at 1731072713723Creating recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp at 1731072713756 (+33 ms)Creating recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp at 1731072713761 (+5 ms)Creating recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp at 1731072713766 (+5 ms)Finishing writing output for hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.1731072526271 so closing down at 1731072713767 (+1 ms)Creating recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp at 1731072713768 (+1 ms)Creating recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp at 1731072713768Creating recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp at 1731072713773 (+5 ms)Creating recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp at 1731072713774 (+1 ms)Creating recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp at 1731072713774Creating recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp at 1731072713781 (+7 ms)3 split writer threads finished at 1731072713782 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1731072713801 (+19 ms)Closed recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731072713803 (+2 ms)Rename recovered edits hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3/recovered.edits/0000000000000000004 at 1731072713803Rename recovered edits hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57/recovered.edits/0000000000000000004 at 1731072713805 (+2 ms)Closed recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1731072713813 (+8 ms)Rename recovered edits hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018/recovered.edits/0000000000000000004 at 1731072713814 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731072713823 (+9 ms)Rename recovered edits hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd/recovered.edits/0000000000000000004 at 1731072713825 (+2 ms)Closed recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731072713833 (+8 ms)Rename recovered edits hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4/recovered.edits/0000000000000000004 at 1731072713835 (+2 ms)Closed recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731072713837 (+2 ms)Closed recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731072713843 (+6 ms)Rename recovered edits hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae/recovered.edits/0000000000000000004 at 1731072713843Rename recovered edits hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe/recovered.edits/0000000000000000004 at 1731072713844 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731072713847 (+3 ms)Rename recovered edits hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38/recovered.edits/0000000000000000004 at 1731072713849 (+2 ms)Closed recovered edits writer path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731072713856 (+7 ms)Rename recovered edits hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe/recovered.edits/0000000000000000004-6ccfea20c990%2C39565%2C1731072524927.1731072526271.temp to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe/recovered.edits/0000000000000000004 at 1731072713858 (+2 ms)Processed 18 edits across 9 Regions in 125 ms; skipped=9; WAL=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.1731072526271, size=327.1 K, length=334920, corrupted=false, cancelled=false at 1731072713858 2024-11-08T13:31:53,859 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.1731072526271 2024-11-08T13:31:53,859 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-08T13:31:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40009 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-08T13:31:53,863 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting/6ccfea20c990%2C39565%2C1731072524927.1731072526271 to hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/oldWALs 2024-11-08T13:31:53,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-08T13:31:53,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.1731072526271, worker=6ccfea20c990,35061,1731072525012 in 310 msec 2024-11-08T13:31:53,869 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:53,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-08T13:31:53,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.1731072526271, worker=6ccfea20c990,35061,1731072525012 in 318 msec 2024-11-08T13:31:53,874 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting dir is empty, no logs to split. 2024-11-08T13:31:53,874 INFO [PEWorker-1 {}] master.SplitWALManager(105): 6ccfea20c990,39565,1731072524927 WAL count=0, meta=false 2024-11-08T13:31:53,874 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 6ccfea20c990,39565,1731072524927 WAL splitting is done? wals=0, meta=false 2024-11-08T13:31:53,875 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-08T13:31:53,875 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:31:53,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:31:53,875 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T13:31:54,049 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 6 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:31:54,049 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T13:31:54,049 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:31:54,049 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T13:31:54,050 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T13:31:54,050 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T13:31:54,050 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T13:31:54,051 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T13:31:54,051 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T13:31:54,053 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T13:31:54,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T13:31:54,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T13:31:54,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T13:31:54,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T13:31:54,057 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T13:31:54,058 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T13:31:54,058 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T13:31:54,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T13:31:54,058 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-08T13:31:54,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=false; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:31:54,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:54,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:54,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T13:31:54,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:54,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.1731072526271, worker=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:54,059 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.1731072526271, worker=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:54,059 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=false; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T13:31:54,059 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:54,059 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:54,059 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:54,059 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:54,060 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.1731072526271, worker=6ccfea20c990,35061,1731072525012 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:54,060 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.1731072526271, worker=6ccfea20c990,35061,1731072525012 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:54,060 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-08T13:31:54,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=61 2024-11-08T13:31:54,060 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ccfea20c990,35367,1731072525051, table=hbase:meta, region=1588230740 2024-11-08T13:31:54,061 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ccfea20c990,35367,1731072525051, state=OPEN 2024-11-08T13:31:54,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:54,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:54,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:54,062 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:54,062 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:54,062 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:54,063 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731072744063 2024-11-08T13:31:54,063 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T13:31:54,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6699738a1be7c8647568269e341753c, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:54,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d5935497231430a7057c66d28a9d018, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:54,069 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7e4837322188444dc48d5c8632c81ce5, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:54,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8fbd70358d0bf3e3f164569432ecfe, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:54,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f0799092106200541bc0c416b0bdbb5, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:54,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d85397130af5b1fc184790336ef5d2b8, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:54,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e1216db90b6f396355f2b6bc308ffe, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:54,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=286367177ffe2bf304003f3b829326ac, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:54,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50c49cba3ff42064b0e071b4b0d3656e, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:54,070 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5970d249471986ac22ae194d250f9a38, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:54,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5503a38fb5ee20ab8673683ba1a1bcaa, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:54,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e178bdc5865404fdcd9e707b6918569c, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:54,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ef0046eead8230ad7213c161e3532bd, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:54,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b082e035d4bc82dc2f93642d66c000c, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:54,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ab60b8cfddafd2fcce990e602b31aef, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:54,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06547784f6904a15d1d127f091988ed3, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:54,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c02aa10a67704d3f15deae347223dc13, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:54,071 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcf86fec97b560cab0f86ca4727e4e57, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:54,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=223bf549b03e7990028b7927dd897a23, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:54,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30f6256cfa21047086ecffea588e9713, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:54,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7f58040b9958dcdcc78f264a8778203, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:54,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00f5b98ab61f3475c8968b8a7a9f6cb4, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:54,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fbf6f08f931761acf639331e9fd9170, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:54,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f973a563f81a494eeb1443f81f0bb069, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:54,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99abbd965e76aa0c5fe5a1011e45a9ae, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:54,072 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1c2472f75aa6a0ed9ba5017889bd71b, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:54,072 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-08T13:31:54,072 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731072774072 2024-11-08T13:31:54,072 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731072834072 2024-11-08T13:31:54,072 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-08T13:31:54,074 INFO [PEWorker-1 {}] master.MasterWalManager(331): Log dir for server 6ccfea20c990,39565,1731072524927 does not exist 2024-11-08T13:31:54,074 INFO [PEWorker-1 {}] master.SplitWALManager(105): 6ccfea20c990,39565,1731072524927 WAL count=0, meta=false 2024-11-08T13:31:54,074 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 6ccfea20c990,39565,1731072524927 WAL splitting is done? wals=0, meta=false 2024-11-08T13:31:54,074 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 6ccfea20c990,39565,1731072524927 failed, ignore...File hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/WALs/6ccfea20c990,39565,1731072524927-splitting does not exist. 2024-11-08T13:31:54,074 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-08T13:31:54,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=00f5b98ab61f3475c8968b8a7a9f6cb4, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5970d249471986ac22ae194d250f9a38, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ef0046eead8230ad7213c161e3532bd, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=99abbd965e76aa0c5fe5a1011e45a9ae, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c4e1216db90b6f396355f2b6bc308ffe, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7c8fbd70358d0bf3e3f164569432ecfe, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4d5935497231430a7057c66d28a9d018, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06547784f6904a15d1d127f091988ed3, ASSIGN}, {pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fcf86fec97b560cab0f86ca4727e4e57, ASSIGN}] 2024-11-08T13:31:54,076 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-08T13:31:54,076 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:31:54,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:31:54,076 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T13:31:54,323 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 7 exec state=pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:31:54,323 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T13:31:54,323 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:31:54,323 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T13:31:54,323 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T13:31:54,324 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T13:31:54,324 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T13:31:54,324 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T13:31:54,325 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T13:31:54,327 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T13:31:54,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T13:31:54,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T13:31:54,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T13:31:54,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T13:31:54,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T13:31:54,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T13:31:54,331 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T13:31:54,332 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T13:31:54,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-08T13:31:54,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=false; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:31:54,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:54,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:54,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T13:31:54,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:54,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.1731072526271, worker=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:54,334 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.1731072526271, worker=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:54,335 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=false; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T13:31:54,335 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:54,335 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:54,335 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:54,335 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:54,335 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.1731072526271, worker=6ccfea20c990,35061,1731072525012 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:54,335 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.1731072526271, worker=6ccfea20c990,35061,1731072525012 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:54,335 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-08T13:31:54,335 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=61 2024-11-08T13:31:54,335 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ccfea20c990,35367,1731072525051, table=hbase:meta, region=1588230740 2024-11-08T13:31:54,336 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ccfea20c990,35367,1731072525051, state=OPEN 2024-11-08T13:31:54,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:54,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:54,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:54,337 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:54,337 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:54,337 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:54,338 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731072744338 2024-11-08T13:31:54,338 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T13:31:54,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6699738a1be7c8647568269e341753c, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:54,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d5935497231430a7057c66d28a9d018, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:54,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7e4837322188444dc48d5c8632c81ce5, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:54,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8fbd70358d0bf3e3f164569432ecfe, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:54,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f0799092106200541bc0c416b0bdbb5, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:54,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d85397130af5b1fc184790336ef5d2b8, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:54,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e1216db90b6f396355f2b6bc308ffe, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:54,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=286367177ffe2bf304003f3b829326ac, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:54,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50c49cba3ff42064b0e071b4b0d3656e, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:54,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5970d249471986ac22ae194d250f9a38, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:54,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5503a38fb5ee20ab8673683ba1a1bcaa, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:54,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e178bdc5865404fdcd9e707b6918569c, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:54,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ef0046eead8230ad7213c161e3532bd, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:54,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b082e035d4bc82dc2f93642d66c000c, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:54,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ab60b8cfddafd2fcce990e602b31aef, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:54,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06547784f6904a15d1d127f091988ed3, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:54,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c02aa10a67704d3f15deae347223dc13, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:54,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcf86fec97b560cab0f86ca4727e4e57, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:54,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=223bf549b03e7990028b7927dd897a23, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:54,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30f6256cfa21047086ecffea588e9713, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:54,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7f58040b9958dcdcc78f264a8778203, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:54,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00f5b98ab61f3475c8968b8a7a9f6cb4, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:54,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fbf6f08f931761acf639331e9fd9170, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:54,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f973a563f81a494eeb1443f81f0bb069, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:54,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99abbd965e76aa0c5fe5a1011e45a9ae, regionState=OPEN, lastHost=6ccfea20c990,39565,1731072524927, regionLocation=6ccfea20c990,39565,1731072524927, openSeqNum=2 2024-11-08T13:31:54,345 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1c2472f75aa6a0ed9ba5017889bd71b, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:54,345 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-08T13:31:54,345 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731072774345 2024-11-08T13:31:54,345 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731072834345 2024-11-08T13:31:54,345 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-08T13:31:54,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=00f5b98ab61f3475c8968b8a7a9f6cb4, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5970d249471986ac22ae194d250f9a38, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ef0046eead8230ad7213c161e3532bd, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=99abbd965e76aa0c5fe5a1011e45a9ae, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c4e1216db90b6f396355f2b6bc308ffe, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7c8fbd70358d0bf3e3f164569432ecfe, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4d5935497231430a7057c66d28a9d018, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06547784f6904a15d1d127f091988ed3, ASSIGN}, {pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fcf86fec97b560cab0f86ca4727e4e57, ASSIGN}] 2024-11-08T13:31:54,345 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-08T13:31:54,347 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fcf86fec97b560cab0f86ca4727e4e57, ASSIGN 2024-11-08T13:31:54,348 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fcf86fec97b560cab0f86ca4727e4e57, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T13:31:54,349 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06547784f6904a15d1d127f091988ed3, ASSIGN 2024-11-08T13:31:54,350 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06547784f6904a15d1d127f091988ed3, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T13:31:54,351 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4d5935497231430a7057c66d28a9d018, ASSIGN 2024-11-08T13:31:54,351 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4d5935497231430a7057c66d28a9d018, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T13:31:54,352 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7c8fbd70358d0bf3e3f164569432ecfe, ASSIGN 2024-11-08T13:31:54,353 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7c8fbd70358d0bf3e3f164569432ecfe, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T13:31:54,353 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c4e1216db90b6f396355f2b6bc308ffe, ASSIGN 2024-11-08T13:31:54,354 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c4e1216db90b6f396355f2b6bc308ffe, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T13:31:54,355 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=99abbd965e76aa0c5fe5a1011e45a9ae, ASSIGN 2024-11-08T13:31:54,355 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=99abbd965e76aa0c5fe5a1011e45a9ae, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T13:31:54,356 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ef0046eead8230ad7213c161e3532bd, ASSIGN 2024-11-08T13:31:54,357 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ef0046eead8230ad7213c161e3532bd, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T13:31:54,357 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5970d249471986ac22ae194d250f9a38, ASSIGN 2024-11-08T13:31:54,358 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5970d249471986ac22ae194d250f9a38, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T13:31:54,358 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=00f5b98ab61f3475c8968b8a7a9f6cb4, ASSIGN 2024-11-08T13:31:54,359 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=00f5b98ab61f3475c8968b8a7a9f6cb4, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T13:31:54,499 DEBUG [6ccfea20c990:40009 {}] balancer.BalancerClusterState(204): Hosts are {6ccfea20c990=0} racks are {/default-rack=0} 2024-11-08T13:31:54,500 DEBUG [6ccfea20c990:40009 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T13:31:54,500 DEBUG [6ccfea20c990:40009 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T13:31:54,500 DEBUG [6ccfea20c990:40009 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T13:31:54,500 DEBUG [6ccfea20c990:40009 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T13:31:54,500 INFO [6ccfea20c990:40009 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T13:31:54,500 INFO [6ccfea20c990:40009 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T13:31:54,500 DEBUG [6ccfea20c990:40009 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T13:31:54,501 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=00f5b98ab61f3475c8968b8a7a9f6cb4, regionState=OPENING, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:54,501 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=06547784f6904a15d1d127f091988ed3, regionState=OPENING, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:54,502 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=5970d249471986ac22ae194d250f9a38, regionState=OPENING, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:54,502 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=7c8fbd70358d0bf3e3f164569432ecfe, regionState=OPENING, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:54,502 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=99abbd965e76aa0c5fe5a1011e45a9ae, regionState=OPENING, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:54,503 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=fcf86fec97b560cab0f86ca4727e4e57, regionState=OPENING, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:54,503 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=5ef0046eead8230ad7213c161e3532bd, regionState=OPENING, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:54,503 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=c4e1216db90b6f396355f2b6bc308ffe, regionState=OPENING, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:54,504 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=00f5b98ab61f3475c8968b8a7a9f6cb4, ASSIGN because future has completed 2024-11-08T13:31:54,504 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=4d5935497231430a7057c66d28a9d018, regionState=OPENING, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:54,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00f5b98ab61f3475c8968b8a7a9f6cb4, server=6ccfea20c990,35367,1731072525051}] 2024-11-08T13:31:54,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06547784f6904a15d1d127f091988ed3, ASSIGN because future has completed 2024-11-08T13:31:54,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5970d249471986ac22ae194d250f9a38, ASSIGN because future has completed 2024-11-08T13:31:54,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7c8fbd70358d0bf3e3f164569432ecfe, ASSIGN because future has completed 2024-11-08T13:31:54,509 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=99abbd965e76aa0c5fe5a1011e45a9ae, ASSIGN because future has completed 2024-11-08T13:31:54,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99abbd965e76aa0c5fe5a1011e45a9ae, server=6ccfea20c990,35061,1731072525012}] 2024-11-08T13:31:54,510 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fcf86fec97b560cab0f86ca4727e4e57, ASSIGN because future has completed 2024-11-08T13:31:54,511 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ef0046eead8230ad7213c161e3532bd, ASSIGN because future has completed 2024-11-08T13:31:54,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c4e1216db90b6f396355f2b6bc308ffe, ASSIGN because future has completed 2024-11-08T13:31:54,519 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4d5935497231430a7057c66d28a9d018, ASSIGN because future has completed 2024-11-08T13:31:54,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d5935497231430a7057c66d28a9d018, server=6ccfea20c990,35061,1731072525012}] 2024-11-08T13:31:54,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4e1216db90b6f396355f2b6bc308ffe, server=6ccfea20c990,35061,1731072525012}] 2024-11-08T13:31:54,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ef0046eead8230ad7213c161e3532bd, server=6ccfea20c990,35061,1731072525012}] 2024-11-08T13:31:54,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure fcf86fec97b560cab0f86ca4727e4e57, server=6ccfea20c990,35061,1731072525012}] 2024-11-08T13:31:54,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c8fbd70358d0bf3e3f164569432ecfe, server=6ccfea20c990,35367,1731072525051}] 2024-11-08T13:31:54,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5970d249471986ac22ae194d250f9a38, server=6ccfea20c990,35367,1731072525051}] 2024-11-08T13:31:54,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06547784f6904a15d1d127f091988ed3, server=6ccfea20c990,35367,1731072525051}] 2024-11-08T13:31:54,662 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe. 2024-11-08T13:31:54,662 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 7c8fbd70358d0bf3e3f164569432ecfe, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-08T13:31:54,662 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:31:54,662 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:31:54,663 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:31:54,663 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:31:54,664 INFO [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:31:54,666 INFO [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c8fbd70358d0bf3e3f164569432ecfe columnFamilyName colfamily11 2024-11-08T13:31:54,666 DEBUG [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,666 INFO [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] regionserver.HStore(327): Store=7c8fbd70358d0bf3e3f164569432ecfe/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,667 INFO [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:31:54,668 INFO [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c8fbd70358d0bf3e3f164569432ecfe columnFamilyName colfamily21 2024-11-08T13:31:54,668 DEBUG [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,668 INFO [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] regionserver.HStore(327): Store=7c8fbd70358d0bf3e3f164569432ecfe/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,668 INFO [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:31:54,670 INFO [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c8fbd70358d0bf3e3f164569432ecfe columnFamilyName colfamily31 2024-11-08T13:31:54,670 DEBUG [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,670 INFO [StoreOpener-7c8fbd70358d0bf3e3f164569432ecfe-1 {}] regionserver.HStore(327): Store=7c8fbd70358d0bf3e3f164569432ecfe/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,670 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:31:54,671 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:31:54,673 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae. 2024-11-08T13:31:54,673 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 99abbd965e76aa0c5fe5a1011e45a9ae, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-08T13:31:54,673 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:31:54,673 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:31:54,673 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:31:54,674 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:31:54,674 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:31:54,674 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe/recovered.edits/0000000000000000004 2024-11-08T13:31:54,675 INFO [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:31:54,676 INFO [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99abbd965e76aa0c5fe5a1011e45a9ae columnFamilyName colfamily11 2024-11-08T13:31:54,676 DEBUG [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,677 INFO [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] regionserver.HStore(327): Store=99abbd965e76aa0c5fe5a1011e45a9ae/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,677 INFO [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:31:54,679 INFO [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99abbd965e76aa0c5fe5a1011e45a9ae columnFamilyName colfamily21 2024-11-08T13:31:54,679 DEBUG [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,679 INFO [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] regionserver.HStore(327): Store=99abbd965e76aa0c5fe5a1011e45a9ae/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,680 INFO [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:31:54,681 INFO [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99abbd965e76aa0c5fe5a1011e45a9ae columnFamilyName colfamily31 2024-11-08T13:31:54,681 DEBUG [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,681 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe/recovered.edits/0000000000000000004 2024-11-08T13:31:54,681 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(2902): Flushing 7c8fbd70358d0bf3e3f164569432ecfe 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:54,681 INFO [StoreOpener-99abbd965e76aa0c5fe5a1011e45a9ae-1 {}] regionserver.HStore(327): Store=99abbd965e76aa0c5fe5a1011e45a9ae/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,682 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:31:54,683 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:31:54,685 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:31:54,686 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae/recovered.edits/0000000000000000004 2024-11-08T13:31:54,693 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae/recovered.edits/0000000000000000004 2024-11-08T13:31:54,693 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(2902): Flushing 99abbd965e76aa0c5fe5a1011e45a9ae 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:54,706 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe/.tmp/colfamily11/6cb989eada694836be2915f78436a860 is 52, key is ddd/colfamily11:colfamily11/1731072530702/Put/seqid=0 2024-11-08T13:31:54,711 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae/.tmp/colfamily11/82f398a7b13147c7876909b3d8431c3a is 52, key is yyy/colfamily11:colfamily11/1731072531009/Put/seqid=0 2024-11-08T13:31:54,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741878_1054 (size=39263) 2024-11-08T13:31:54,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741878_1054 (size=39263) 2024-11-08T13:31:54,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741878_1054 (size=39263) 2024-11-08T13:31:54,720 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe/.tmp/colfamily11/6cb989eada694836be2915f78436a860 2024-11-08T13:31:54,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741879_1055 (size=39263) 2024-11-08T13:31:54,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741879_1055 (size=39263) 2024-11-08T13:31:54,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741879_1055 (size=39263) 2024-11-08T13:31:54,725 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae/.tmp/colfamily11/82f398a7b13147c7876909b3d8431c3a 2024-11-08T13:31:54,731 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe/.tmp/colfamily11/6cb989eada694836be2915f78436a860 as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe/colfamily11/6cb989eada694836be2915f78436a860 2024-11-08T13:31:54,734 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae/.tmp/colfamily11/82f398a7b13147c7876909b3d8431c3a as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae/colfamily11/82f398a7b13147c7876909b3d8431c3a 2024-11-08T13:31:54,738 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe/colfamily11/6cb989eada694836be2915f78436a860, entries=703, sequenceid=4, filesize=38.3 K 2024-11-08T13:31:54,739 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 7c8fbd70358d0bf3e3f164569432ecfe in 57ms, sequenceid=4, compaction requested=false; wal=null 2024-11-08T13:31:54,739 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe/recovered.edits/0000000000000000004 2024-11-08T13:31:54,741 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:31:54,741 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:31:54,742 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae/colfamily11/82f398a7b13147c7876909b3d8431c3a, entries=703, sequenceid=4, filesize=38.3 K 2024-11-08T13:31:54,742 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:31:54,742 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 99abbd965e76aa0c5fe5a1011e45a9ae in 49ms, sequenceid=4, compaction requested=false; wal=null 2024-11-08T13:31:54,743 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae/recovered.edits/0000000000000000004 2024-11-08T13:31:54,744 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:31:54,744 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:31:54,744 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:31:54,744 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:31:54,746 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:31:54,746 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T13:31:54,747 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 7c8fbd70358d0bf3e3f164569432ecfe; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73640564, jitterRate=0.09732991456985474}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:31:54,748 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:31:54,748 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 7c8fbd70358d0bf3e3f164569432ecfe: Running coprocessor pre-open hook at 1731072714663Writing region info on filesystem at 1731072714663Initializing all the Stores at 1731072714664 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714664Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714664Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714664Obtaining lock to block concurrent updates at 1731072714681 (+17 ms)Preparing flush snapshotting stores in 7c8fbd70358d0bf3e3f164569432ecfe at 1731072714681Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072714681Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe. at 1731072714681Flushing 7c8fbd70358d0bf3e3f164569432ecfe/colfamily11: creating writer at 1731072714681Flushing 7c8fbd70358d0bf3e3f164569432ecfe/colfamily11: appending metadata at 1731072714706 (+25 ms)Flushing 7c8fbd70358d0bf3e3f164569432ecfe/colfamily11: closing flushed file at 1731072714706Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a243ca4: reopening flushed file at 1731072714729 (+23 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 7c8fbd70358d0bf3e3f164569432ecfe in 57ms, sequenceid=4, compaction requested=false; wal=null at 1731072714739 (+10 ms)Cleaning up temporary data from old regions at 1731072714741 (+2 ms)Running coprocessor post-open hooks at 1731072714748 (+7 ms)Region opened successfully at 1731072714748 2024-11-08T13:31:54,748 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T13:31:54,749 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe., pid=79, masterSystemTime=1731072714658 2024-11-08T13:31:54,749 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened 99abbd965e76aa0c5fe5a1011e45a9ae; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60117325, jitterRate=-0.10418204963207245}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:31:54,750 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:31:54,750 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 99abbd965e76aa0c5fe5a1011e45a9ae: Running coprocessor pre-open hook at 1731072714674Writing region info on filesystem at 1731072714674Initializing all the Stores at 1731072714675 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714675Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714675Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714675Obtaining lock to block concurrent updates at 1731072714693 (+18 ms)Preparing flush snapshotting stores in 99abbd965e76aa0c5fe5a1011e45a9ae at 1731072714693Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072714693Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae. at 1731072714693Flushing 99abbd965e76aa0c5fe5a1011e45a9ae/colfamily11: creating writer at 1731072714693Flushing 99abbd965e76aa0c5fe5a1011e45a9ae/colfamily11: appending metadata at 1731072714711 (+18 ms)Flushing 99abbd965e76aa0c5fe5a1011e45a9ae/colfamily11: closing flushed file at 1731072714711Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48b2e18f: reopening flushed file at 1731072714733 (+22 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 99abbd965e76aa0c5fe5a1011e45a9ae in 49ms, sequenceid=4, compaction requested=false; wal=null at 1731072714742 (+9 ms)Cleaning up temporary data from old regions at 1731072714744 (+2 ms)Running coprocessor post-open hooks at 1731072714750 (+6 ms)Region opened successfully at 1731072714750 2024-11-08T13:31:54,751 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae., pid=74, masterSystemTime=1731072714668 2024-11-08T13:31:54,751 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe. 2024-11-08T13:31:54,751 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe. 2024-11-08T13:31:54,751 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38. 2024-11-08T13:31:54,751 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 5970d249471986ac22ae194d250f9a38, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-08T13:31:54,752 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 5970d249471986ac22ae194d250f9a38 2024-11-08T13:31:54,752 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:31:54,752 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 5970d249471986ac22ae194d250f9a38 2024-11-08T13:31:54,752 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 5970d249471986ac22ae194d250f9a38 2024-11-08T13:31:54,752 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=7c8fbd70358d0bf3e3f164569432ecfe, regionState=OPEN, openSeqNum=5, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:54,752 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae. 2024-11-08T13:31:54,752 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae. 2024-11-08T13:31:54,753 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd. 2024-11-08T13:31:54,753 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 5ef0046eead8230ad7213c161e3532bd, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-08T13:31:54,753 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:31:54,753 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:31:54,753 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:31:54,753 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:31:54,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c8fbd70358d0bf3e3f164569432ecfe, server=6ccfea20c990,35367,1731072525051 because future has completed 2024-11-08T13:31:54,755 INFO [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 5970d249471986ac22ae194d250f9a38 2024-11-08T13:31:54,755 INFO [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:31:54,755 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=99abbd965e76aa0c5fe5a1011e45a9ae, regionState=OPEN, openSeqNum=5, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:54,757 INFO [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5970d249471986ac22ae194d250f9a38 columnFamilyName colfamily11 2024-11-08T13:31:54,757 DEBUG [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,757 INFO [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ef0046eead8230ad7213c161e3532bd columnFamilyName colfamily11 2024-11-08T13:31:54,757 DEBUG [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,758 INFO [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] regionserver.HStore(327): Store=5970d249471986ac22ae194d250f9a38/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,758 INFO [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 5970d249471986ac22ae194d250f9a38 2024-11-08T13:31:54,758 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99abbd965e76aa0c5fe5a1011e45a9ae, server=6ccfea20c990,35061,1731072525012 because future has completed 2024-11-08T13:31:54,759 INFO [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] regionserver.HStore(327): Store=5ef0046eead8230ad7213c161e3532bd/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,759 INFO [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5970d249471986ac22ae194d250f9a38 columnFamilyName colfamily21 2024-11-08T13:31:54,759 INFO [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:31:54,759 DEBUG [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,760 INFO [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] regionserver.HStore(327): Store=5970d249471986ac22ae194d250f9a38/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,760 INFO [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 5970d249471986ac22ae194d250f9a38 2024-11-08T13:31:54,760 INFO [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ef0046eead8230ad7213c161e3532bd columnFamilyName colfamily21 2024-11-08T13:31:54,761 DEBUG [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,761 INFO [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5970d249471986ac22ae194d250f9a38 columnFamilyName colfamily31 2024-11-08T13:31:54,761 DEBUG [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,761 INFO [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] regionserver.HStore(327): Store=5ef0046eead8230ad7213c161e3532bd/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,761 INFO [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:31:54,761 INFO [StoreOpener-5970d249471986ac22ae194d250f9a38-1 {}] regionserver.HStore(327): Store=5970d249471986ac22ae194d250f9a38/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,762 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 5970d249471986ac22ae194d250f9a38 2024-11-08T13:31:54,762 INFO [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ef0046eead8230ad7213c161e3532bd columnFamilyName colfamily31 2024-11-08T13:31:54,762 DEBUG [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,763 INFO [StoreOpener-5ef0046eead8230ad7213c161e3532bd-1 {}] regionserver.HStore(327): Store=5ef0046eead8230ad7213c161e3532bd/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,763 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38 2024-11-08T13:31:54,763 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:31:54,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=69 2024-11-08T13:31:54,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 7c8fbd70358d0bf3e3f164569432ecfe, server=6ccfea20c990,35367,1731072525051 in 226 msec 2024-11-08T13:31:54,764 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:31:54,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7c8fbd70358d0bf3e3f164569432ecfe, ASSIGN in 418 msec 2024-11-08T13:31:54,765 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38 2024-11-08T13:31:54,766 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38/recovered.edits/0000000000000000004 2024-11-08T13:31:54,766 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:31:54,768 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd/recovered.edits/0000000000000000004 2024-11-08T13:31:54,770 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=67 2024-11-08T13:31:54,770 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure 99abbd965e76aa0c5fe5a1011e45a9ae, server=6ccfea20c990,35061,1731072525012 in 257 msec 2024-11-08T13:31:54,772 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38/recovered.edits/0000000000000000004 2024-11-08T13:31:54,772 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(2902): Flushing 5970d249471986ac22ae194d250f9a38 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:54,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=99abbd965e76aa0c5fe5a1011e45a9ae, ASSIGN in 425 msec 2024-11-08T13:31:54,777 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd/recovered.edits/0000000000000000004 2024-11-08T13:31:54,778 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(2902): Flushing 5ef0046eead8230ad7213c161e3532bd 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:54,791 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38/.tmp/colfamily11/915373c554394c1b9110a080af11dadc is 52, key is jjj/colfamily11:colfamily11/1731072530818/Put/seqid=0 2024-11-08T13:31:54,794 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd/.tmp/colfamily11/fd1046c453144f4c8bd6cbe5cdd72629 is 52, key is mmm/colfamily11:colfamily11/1731072530856/Put/seqid=0 2024-11-08T13:31:54,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741880_1056 (size=39263) 2024-11-08T13:31:54,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741880_1056 (size=39263) 2024-11-08T13:31:54,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741880_1056 (size=39263) 2024-11-08T13:31:54,803 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38/.tmp/colfamily11/915373c554394c1b9110a080af11dadc 2024-11-08T13:31:54,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741881_1057 (size=39263) 2024-11-08T13:31:54,811 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38/.tmp/colfamily11/915373c554394c1b9110a080af11dadc as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38/colfamily11/915373c554394c1b9110a080af11dadc 2024-11-08T13:31:54,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741881_1057 (size=39263) 2024-11-08T13:31:54,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741881_1057 (size=39263) 2024-11-08T13:31:54,814 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd/.tmp/colfamily11/fd1046c453144f4c8bd6cbe5cdd72629 2024-11-08T13:31:54,820 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38/colfamily11/915373c554394c1b9110a080af11dadc, entries=703, sequenceid=4, filesize=38.3 K 2024-11-08T13:31:54,821 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5970d249471986ac22ae194d250f9a38 in 49ms, sequenceid=4, compaction requested=false; wal=null 2024-11-08T13:31:54,822 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38/recovered.edits/0000000000000000004 2024-11-08T13:31:54,822 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd/.tmp/colfamily11/fd1046c453144f4c8bd6cbe5cdd72629 as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd/colfamily11/fd1046c453144f4c8bd6cbe5cdd72629 2024-11-08T13:31:54,823 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 5970d249471986ac22ae194d250f9a38 2024-11-08T13:31:54,823 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 5970d249471986ac22ae194d250f9a38 2024-11-08T13:31:54,824 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:31:54,826 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 5970d249471986ac22ae194d250f9a38 2024-11-08T13:31:54,828 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T13:31:54,829 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 5970d249471986ac22ae194d250f9a38; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73905468, jitterRate=0.10127729177474976}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:31:54,829 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd/colfamily11/fd1046c453144f4c8bd6cbe5cdd72629, entries=703, sequenceid=4, filesize=38.3 K 2024-11-08T13:31:54,829 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5970d249471986ac22ae194d250f9a38 2024-11-08T13:31:54,830 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 5970d249471986ac22ae194d250f9a38: Running coprocessor pre-open hook at 1731072714752Writing region info on filesystem at 1731072714752Initializing all the Stores at 1731072714754 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714754Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714755 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714755Obtaining lock to block concurrent updates at 1731072714772 (+17 ms)Preparing flush snapshotting stores in 5970d249471986ac22ae194d250f9a38 at 1731072714772Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072714773 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38. at 1731072714773Flushing 5970d249471986ac22ae194d250f9a38/colfamily11: creating writer at 1731072714773Flushing 5970d249471986ac22ae194d250f9a38/colfamily11: appending metadata at 1731072714790 (+17 ms)Flushing 5970d249471986ac22ae194d250f9a38/colfamily11: closing flushed file at 1731072714790Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49b49da5: reopening flushed file at 1731072714810 (+20 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5970d249471986ac22ae194d250f9a38 in 49ms, sequenceid=4, compaction requested=false; wal=null at 1731072714821 (+11 ms)Cleaning up temporary data from old regions at 1731072714823 (+2 ms)Running coprocessor post-open hooks at 1731072714829 (+6 ms)Region opened successfully at 1731072714829 2024-11-08T13:31:54,830 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5ef0046eead8230ad7213c161e3532bd in 53ms, sequenceid=4, compaction requested=false; wal=null 2024-11-08T13:31:54,830 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38., pid=80, masterSystemTime=1731072714658 2024-11-08T13:31:54,831 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd/recovered.edits/0000000000000000004 2024-11-08T13:31:54,832 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:31:54,832 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:31:54,832 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38. 2024-11-08T13:31:54,832 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38. 2024-11-08T13:31:54,832 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3. 2024-11-08T13:31:54,832 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 06547784f6904a15d1d127f091988ed3, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-08T13:31:54,833 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 06547784f6904a15d1d127f091988ed3 2024-11-08T13:31:54,833 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:31:54,833 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 06547784f6904a15d1d127f091988ed3 2024-11-08T13:31:54,833 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 06547784f6904a15d1d127f091988ed3 2024-11-08T13:31:54,833 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:31:54,834 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=5970d249471986ac22ae194d250f9a38, regionState=OPEN, openSeqNum=5, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:54,836 INFO [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 06547784f6904a15d1d127f091988ed3 2024-11-08T13:31:54,836 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:31:54,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5970d249471986ac22ae194d250f9a38, server=6ccfea20c990,35367,1731072525051 because future has completed 2024-11-08T13:31:54,837 INFO [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06547784f6904a15d1d127f091988ed3 columnFamilyName colfamily11 2024-11-08T13:31:54,837 DEBUG [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,837 INFO [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] regionserver.HStore(327): Store=06547784f6904a15d1d127f091988ed3/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,838 INFO [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 06547784f6904a15d1d127f091988ed3 2024-11-08T13:31:54,839 INFO [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06547784f6904a15d1d127f091988ed3 columnFamilyName colfamily21 2024-11-08T13:31:54,839 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T13:31:54,839 DEBUG [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,840 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 5ef0046eead8230ad7213c161e3532bd; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70184951, jitterRate=0.045837268233299255}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:31:54,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=65 2024-11-08T13:31:54,840 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:31:54,840 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure 5970d249471986ac22ae194d250f9a38, server=6ccfea20c990,35367,1731072525051 in 301 msec 2024-11-08T13:31:54,840 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 5ef0046eead8230ad7213c161e3532bd: Running coprocessor pre-open hook at 1731072714753Writing region info on filesystem at 1731072714753Initializing all the Stores at 1731072714755 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714755Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714755Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714755Obtaining lock to block concurrent updates at 1731072714778 (+23 ms)Preparing flush snapshotting stores in 5ef0046eead8230ad7213c161e3532bd at 1731072714778Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072714778Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd. at 1731072714778Flushing 5ef0046eead8230ad7213c161e3532bd/colfamily11: creating writer at 1731072714778Flushing 5ef0046eead8230ad7213c161e3532bd/colfamily11: appending metadata at 1731072714794 (+16 ms)Flushing 5ef0046eead8230ad7213c161e3532bd/colfamily11: closing flushed file at 1731072714794Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7994e468: reopening flushed file at 1731072714821 (+27 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5ef0046eead8230ad7213c161e3532bd in 53ms, sequenceid=4, compaction requested=false; wal=null at 1731072714830 (+9 ms)Cleaning up temporary data from old regions at 1731072714832 (+2 ms)Running coprocessor post-open hooks at 1731072714840 (+8 ms)Region opened successfully at 1731072714840 2024-11-08T13:31:54,841 INFO [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] regionserver.HStore(327): Store=06547784f6904a15d1d127f091988ed3/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,841 INFO [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 06547784f6904a15d1d127f091988ed3 2024-11-08T13:31:54,841 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd., pid=77, masterSystemTime=1731072714668 2024-11-08T13:31:54,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5970d249471986ac22ae194d250f9a38, ASSIGN in 495 msec 2024-11-08T13:31:54,842 INFO [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06547784f6904a15d1d127f091988ed3 columnFamilyName colfamily31 2024-11-08T13:31:54,842 DEBUG [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,843 INFO [StoreOpener-06547784f6904a15d1d127f091988ed3-1 {}] regionserver.HStore(327): Store=06547784f6904a15d1d127f091988ed3/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,843 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd. 2024-11-08T13:31:54,843 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 06547784f6904a15d1d127f091988ed3 2024-11-08T13:31:54,843 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd. 2024-11-08T13:31:54,843 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018. 2024-11-08T13:31:54,843 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 4d5935497231430a7057c66d28a9d018, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-08T13:31:54,843 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=5ef0046eead8230ad7213c161e3532bd, regionState=OPEN, openSeqNum=5, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:54,844 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 4d5935497231430a7057c66d28a9d018 2024-11-08T13:31:54,844 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:31:54,844 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3 2024-11-08T13:31:54,844 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 4d5935497231430a7057c66d28a9d018 2024-11-08T13:31:54,844 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 4d5935497231430a7057c66d28a9d018 2024-11-08T13:31:54,846 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ef0046eead8230ad7213c161e3532bd, server=6ccfea20c990,35061,1731072525012 because future has completed 2024-11-08T13:31:54,846 INFO [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 4d5935497231430a7057c66d28a9d018 2024-11-08T13:31:54,846 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3 2024-11-08T13:31:54,847 INFO [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d5935497231430a7057c66d28a9d018 columnFamilyName colfamily11 2024-11-08T13:31:54,847 DEBUG [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,848 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3/recovered.edits/0000000000000000004 2024-11-08T13:31:54,848 INFO [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] regionserver.HStore(327): Store=4d5935497231430a7057c66d28a9d018/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,848 INFO [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 4d5935497231430a7057c66d28a9d018 2024-11-08T13:31:54,849 INFO [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d5935497231430a7057c66d28a9d018 columnFamilyName colfamily21 2024-11-08T13:31:54,849 DEBUG [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,850 INFO [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] regionserver.HStore(327): Store=4d5935497231430a7057c66d28a9d018/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=66 2024-11-08T13:31:54,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 5ef0046eead8230ad7213c161e3532bd, server=6ccfea20c990,35061,1731072525012 in 321 msec 2024-11-08T13:31:54,850 INFO [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 4d5935497231430a7057c66d28a9d018 2024-11-08T13:31:54,851 INFO [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d5935497231430a7057c66d28a9d018 columnFamilyName colfamily31 2024-11-08T13:31:54,851 DEBUG [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,852 INFO [StoreOpener-4d5935497231430a7057c66d28a9d018-1 {}] regionserver.HStore(327): Store=4d5935497231430a7057c66d28a9d018/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,852 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 4d5935497231430a7057c66d28a9d018 2024-11-08T13:31:54,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ef0046eead8230ad7213c161e3532bd, ASSIGN in 505 msec 2024-11-08T13:31:54,853 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018 2024-11-08T13:31:54,854 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3/recovered.edits/0000000000000000004 2024-11-08T13:31:54,855 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(2902): Flushing 06547784f6904a15d1d127f091988ed3 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:54,855 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018 2024-11-08T13:31:54,856 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018/recovered.edits/0000000000000000004 2024-11-08T13:31:54,861 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018/recovered.edits/0000000000000000004 2024-11-08T13:31:54,861 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(2902): Flushing 4d5935497231430a7057c66d28a9d018 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:54,877 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3/.tmp/colfamily11/d3e35b3ade884e7a90db459af8c3acee is 52, key is ppp/colfamily11:colfamily11/1731072530895/Put/seqid=0 2024-11-08T13:31:54,877 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018/.tmp/colfamily11/e87b3b9c632d42f683ac89a260786b58 is 52, key is bbb/colfamily11:colfamily11/1731072530669/Put/seqid=0 2024-11-08T13:31:54,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741882_1058 (size=39263) 2024-11-08T13:31:54,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741883_1059 (size=39263) 2024-11-08T13:31:54,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741882_1058 (size=39263) 2024-11-08T13:31:54,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741883_1059 (size=39263) 2024-11-08T13:31:54,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741883_1059 (size=39263) 2024-11-08T13:31:54,891 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018/.tmp/colfamily11/e87b3b9c632d42f683ac89a260786b58 2024-11-08T13:31:54,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741882_1058 (size=39263) 2024-11-08T13:31:54,891 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3/.tmp/colfamily11/d3e35b3ade884e7a90db459af8c3acee 2024-11-08T13:31:54,901 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3/.tmp/colfamily11/d3e35b3ade884e7a90db459af8c3acee as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3/colfamily11/d3e35b3ade884e7a90db459af8c3acee 2024-11-08T13:31:54,903 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018/.tmp/colfamily11/e87b3b9c632d42f683ac89a260786b58 as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018/colfamily11/e87b3b9c632d42f683ac89a260786b58 2024-11-08T13:31:54,909 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3/colfamily11/d3e35b3ade884e7a90db459af8c3acee, entries=703, sequenceid=4, filesize=38.3 K 2024-11-08T13:31:54,909 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018/colfamily11/e87b3b9c632d42f683ac89a260786b58, entries=703, sequenceid=4, filesize=38.3 K 2024-11-08T13:31:54,910 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 4d5935497231430a7057c66d28a9d018 in 48ms, sequenceid=4, compaction requested=false; wal=null 2024-11-08T13:31:54,910 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 06547784f6904a15d1d127f091988ed3 in 54ms, sequenceid=4, compaction requested=false; wal=null 2024-11-08T13:31:54,910 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018/recovered.edits/0000000000000000004 2024-11-08T13:31:54,910 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3/recovered.edits/0000000000000000004 2024-11-08T13:31:54,911 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 4d5935497231430a7057c66d28a9d018 2024-11-08T13:31:54,911 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 4d5935497231430a7057c66d28a9d018 2024-11-08T13:31:54,911 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 06547784f6904a15d1d127f091988ed3 2024-11-08T13:31:54,911 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 06547784f6904a15d1d127f091988ed3 2024-11-08T13:31:54,912 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:31:54,912 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:31:54,913 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 06547784f6904a15d1d127f091988ed3 2024-11-08T13:31:54,913 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 4d5935497231430a7057c66d28a9d018 2024-11-08T13:31:54,915 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T13:31:54,915 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T13:31:54,916 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 4d5935497231430a7057c66d28a9d018; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60490432, jitterRate=-0.09862232208251953}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:31:54,916 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4d5935497231430a7057c66d28a9d018 2024-11-08T13:31:54,916 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 06547784f6904a15d1d127f091988ed3; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63000916, jitterRate=-0.06121319532394409}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:31:54,916 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06547784f6904a15d1d127f091988ed3 2024-11-08T13:31:54,916 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 4d5935497231430a7057c66d28a9d018: Running coprocessor pre-open hook at 1731072714844Writing region info on filesystem at 1731072714844Initializing all the Stores at 1731072714846 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714846Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714846Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714846Obtaining lock to block concurrent updates at 1731072714861 (+15 ms)Preparing flush snapshotting stores in 4d5935497231430a7057c66d28a9d018 at 1731072714861Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072714861Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018. at 1731072714861Flushing 4d5935497231430a7057c66d28a9d018/colfamily11: creating writer at 1731072714861Flushing 4d5935497231430a7057c66d28a9d018/colfamily11: appending metadata at 1731072714877 (+16 ms)Flushing 4d5935497231430a7057c66d28a9d018/colfamily11: closing flushed file at 1731072714877Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2283d3f5: reopening flushed file at 1731072714901 (+24 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 4d5935497231430a7057c66d28a9d018 in 48ms, sequenceid=4, compaction requested=false; wal=null at 1731072714910 (+9 ms)Cleaning up temporary data from old regions at 1731072714911 (+1 ms)Running coprocessor post-open hooks at 1731072714916 (+5 ms)Region opened successfully at 1731072714916 2024-11-08T13:31:54,916 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 06547784f6904a15d1d127f091988ed3: Running coprocessor pre-open hook at 1731072714833Writing region info on filesystem at 1731072714833Initializing all the Stores at 1731072714835 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714835Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714835Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714835Obtaining lock to block concurrent updates at 1731072714855 (+20 ms)Preparing flush snapshotting stores in 06547784f6904a15d1d127f091988ed3 at 1731072714855Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072714855Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3. at 1731072714855Flushing 06547784f6904a15d1d127f091988ed3/colfamily11: creating writer at 1731072714855Flushing 06547784f6904a15d1d127f091988ed3/colfamily11: appending metadata at 1731072714877 (+22 ms)Flushing 06547784f6904a15d1d127f091988ed3/colfamily11: closing flushed file at 1731072714877Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ffcb2f3: reopening flushed file at 1731072714900 (+23 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 06547784f6904a15d1d127f091988ed3 in 54ms, sequenceid=4, compaction requested=false; wal=null at 1731072714910 (+10 ms)Cleaning up temporary data from old regions at 1731072714911 (+1 ms)Running coprocessor post-open hooks at 1731072714916 (+5 ms)Region opened successfully at 1731072714916 2024-11-08T13:31:54,917 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018., pid=75, masterSystemTime=1731072714668 2024-11-08T13:31:54,917 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3., pid=81, masterSystemTime=1731072714658 2024-11-08T13:31:54,918 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018. 2024-11-08T13:31:54,918 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018. 2024-11-08T13:31:54,918 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57. 2024-11-08T13:31:54,919 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => fcf86fec97b560cab0f86ca4727e4e57, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-08T13:31:54,919 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=4d5935497231430a7057c66d28a9d018, regionState=OPEN, openSeqNum=5, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:54,919 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3. 2024-11-08T13:31:54,919 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3. 2024-11-08T13:31:54,919 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:31:54,919 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4. 2024-11-08T13:31:54,919 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:31:54,919 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:31:54,919 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => 00f5b98ab61f3475c8968b8a7a9f6cb4, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-08T13:31:54,919 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:31:54,919 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:31:54,920 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:31:54,920 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:31:54,920 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:31:54,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d5935497231430a7057c66d28a9d018, server=6ccfea20c990,35061,1731072525012 because future has completed 2024-11-08T13:31:54,921 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=06547784f6904a15d1d127f091988ed3, regionState=OPEN, openSeqNum=5, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:54,922 INFO [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:31:54,922 INFO [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:31:54,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06547784f6904a15d1d127f091988ed3, server=6ccfea20c990,35367,1731072525051 because future has completed 2024-11-08T13:31:54,923 INFO [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00f5b98ab61f3475c8968b8a7a9f6cb4 columnFamilyName colfamily11 2024-11-08T13:31:54,923 INFO [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fcf86fec97b560cab0f86ca4727e4e57 columnFamilyName colfamily11 2024-11-08T13:31:54,923 DEBUG [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,923 DEBUG [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,923 INFO [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] regionserver.HStore(327): Store=fcf86fec97b560cab0f86ca4727e4e57/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,924 INFO [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:31:54,924 INFO [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] regionserver.HStore(327): Store=00f5b98ab61f3475c8968b8a7a9f6cb4/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,924 INFO [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:31:54,925 INFO [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fcf86fec97b560cab0f86ca4727e4e57 columnFamilyName colfamily21 2024-11-08T13:31:54,925 DEBUG [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,925 INFO [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] regionserver.HStore(327): Store=fcf86fec97b560cab0f86ca4727e4e57/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,925 INFO [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:31:54,926 INFO [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00f5b98ab61f3475c8968b8a7a9f6cb4 columnFamilyName colfamily21 2024-11-08T13:31:54,926 DEBUG [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=70 2024-11-08T13:31:54,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure 4d5935497231430a7057c66d28a9d018, server=6ccfea20c990,35061,1731072525012 in 403 msec 2024-11-08T13:31:54,926 INFO [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] regionserver.HStore(327): Store=00f5b98ab61f3475c8968b8a7a9f6cb4/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,926 INFO [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fcf86fec97b560cab0f86ca4727e4e57 columnFamilyName colfamily31 2024-11-08T13:31:54,926 INFO [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:31:54,926 DEBUG [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,927 INFO [StoreOpener-fcf86fec97b560cab0f86ca4727e4e57-1 {}] regionserver.HStore(327): Store=fcf86fec97b560cab0f86ca4727e4e57/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,927 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:31:54,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4d5935497231430a7057c66d28a9d018, ASSIGN in 581 msec 2024-11-08T13:31:54,928 INFO [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00f5b98ab61f3475c8968b8a7a9f6cb4 columnFamilyName colfamily31 2024-11-08T13:31:54,929 DEBUG [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:54,929 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:31:54,929 INFO [StoreOpener-00f5b98ab61f3475c8968b8a7a9f6cb4-1 {}] regionserver.HStore(327): Store=00f5b98ab61f3475c8968b8a7a9f6cb4/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:54,929 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:31:54,930 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:31:54,931 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:31:54,931 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57/recovered.edits/0000000000000000004 2024-11-08T13:31:54,932 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:31:54,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=71 2024-11-08T13:31:54,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure 06547784f6904a15d1d127f091988ed3, server=6ccfea20c990,35367,1731072525051 in 389 msec 2024-11-08T13:31:54,933 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4/recovered.edits/0000000000000000004 2024-11-08T13:31:54,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06547784f6904a15d1d127f091988ed3, ASSIGN in 588 msec 2024-11-08T13:31:54,938 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4/recovered.edits/0000000000000000004 2024-11-08T13:31:54,938 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(2902): Flushing 00f5b98ab61f3475c8968b8a7a9f6cb4 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:54,938 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57/recovered.edits/0000000000000000004 2024-11-08T13:31:54,938 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(2902): Flushing fcf86fec97b560cab0f86ca4727e4e57 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:54,953 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4/.tmp/colfamily11/09d2d98539c64d30a1646962b8cbe72e is 52, key is vvv/colfamily11:colfamily11/1731072530971/Put/seqid=0 2024-11-08T13:31:54,959 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57/.tmp/colfamily11/e2889f9286344e8687135baf26306f01 is 52, key is rrr/colfamily11:colfamily11/1731072530933/Put/seqid=0 2024-11-08T13:31:54,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741884_1060 (size=39263) 2024-11-08T13:31:54,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741884_1060 (size=39263) 2024-11-08T13:31:54,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741884_1060 (size=39263) 2024-11-08T13:31:54,965 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4/.tmp/colfamily11/09d2d98539c64d30a1646962b8cbe72e 2024-11-08T13:31:54,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741885_1061 (size=39263) 2024-11-08T13:31:54,974 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57/.tmp/colfamily11/e2889f9286344e8687135baf26306f01 2024-11-08T13:31:54,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741885_1061 (size=39263) 2024-11-08T13:31:54,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741885_1061 (size=39263) 2024-11-08T13:31:54,975 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4/.tmp/colfamily11/09d2d98539c64d30a1646962b8cbe72e as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4/colfamily11/09d2d98539c64d30a1646962b8cbe72e 2024-11-08T13:31:54,982 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57/.tmp/colfamily11/e2889f9286344e8687135baf26306f01 as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57/colfamily11/e2889f9286344e8687135baf26306f01 2024-11-08T13:31:54,983 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4/colfamily11/09d2d98539c64d30a1646962b8cbe72e, entries=703, sequenceid=4, filesize=38.3 K 2024-11-08T13:31:54,983 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 00f5b98ab61f3475c8968b8a7a9f6cb4 in 45ms, sequenceid=4, compaction requested=false; wal=null 2024-11-08T13:31:54,984 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4/recovered.edits/0000000000000000004 2024-11-08T13:31:54,986 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:31:54,986 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:31:54,987 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:31:54,989 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:31:54,991 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57/colfamily11/e2889f9286344e8687135baf26306f01, entries=703, sequenceid=4, filesize=38.3 K 2024-11-08T13:31:54,991 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T13:31:54,991 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for fcf86fec97b560cab0f86ca4727e4e57 in 53ms, sequenceid=4, compaction requested=false; wal=null 2024-11-08T13:31:54,992 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57/recovered.edits/0000000000000000004 2024-11-08T13:31:54,992 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened 00f5b98ab61f3475c8968b8a7a9f6cb4; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64508798, jitterRate=-0.0387440025806427}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:31:54,992 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:31:54,992 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for 00f5b98ab61f3475c8968b8a7a9f6cb4: Running coprocessor pre-open hook at 1731072714920Writing region info on filesystem at 1731072714920Initializing all the Stores at 1731072714921 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714921Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714922 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714922Obtaining lock to block concurrent updates at 1731072714938 (+16 ms)Preparing flush snapshotting stores in 00f5b98ab61f3475c8968b8a7a9f6cb4 at 1731072714938Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072714938Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4. at 1731072714938Flushing 00f5b98ab61f3475c8968b8a7a9f6cb4/colfamily11: creating writer at 1731072714938Flushing 00f5b98ab61f3475c8968b8a7a9f6cb4/colfamily11: appending metadata at 1731072714953 (+15 ms)Flushing 00f5b98ab61f3475c8968b8a7a9f6cb4/colfamily11: closing flushed file at 1731072714953Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@240a294b: reopening flushed file at 1731072714973 (+20 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 00f5b98ab61f3475c8968b8a7a9f6cb4 in 45ms, sequenceid=4, compaction requested=false; wal=null at 1731072714983 (+10 ms)Cleaning up temporary data from old regions at 1731072714986 (+3 ms)Running coprocessor post-open hooks at 1731072714992 (+6 ms)Region opened successfully at 1731072714992 2024-11-08T13:31:54,993 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:31:54,993 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:31:54,993 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4., pid=73, masterSystemTime=1731072714658 2024-11-08T13:31:54,993 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:31:54,995 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:31:54,995 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4. 2024-11-08T13:31:54,996 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4. 2024-11-08T13:31:54,996 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=00f5b98ab61f3475c8968b8a7a9f6cb4, regionState=OPEN, openSeqNum=5, regionLocation=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:54,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00f5b98ab61f3475c8968b8a7a9f6cb4, server=6ccfea20c990,35367,1731072525051 because future has completed 2024-11-08T13:31:55,001 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T13:31:55,002 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened fcf86fec97b560cab0f86ca4727e4e57; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61779499, jitterRate=-0.07941372692584991}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:31:55,003 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:31:55,003 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for fcf86fec97b560cab0f86ca4727e4e57: Running coprocessor pre-open hook at 1731072714919Writing region info on filesystem at 1731072714919Initializing all the Stores at 1731072714920 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714920Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714921 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072714922 (+1 ms)Obtaining lock to block concurrent updates at 1731072714938 (+16 ms)Preparing flush snapshotting stores in fcf86fec97b560cab0f86ca4727e4e57 at 1731072714938Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072714938Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57. at 1731072714938Flushing fcf86fec97b560cab0f86ca4727e4e57/colfamily11: creating writer at 1731072714939 (+1 ms)Flushing fcf86fec97b560cab0f86ca4727e4e57/colfamily11: appending metadata at 1731072714959 (+20 ms)Flushing fcf86fec97b560cab0f86ca4727e4e57/colfamily11: closing flushed file at 1731072714959Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50b7c811: reopening flushed file at 1731072714981 (+22 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for fcf86fec97b560cab0f86ca4727e4e57 in 53ms, sequenceid=4, compaction requested=false; wal=null at 1731072714991 (+10 ms)Cleaning up temporary data from old regions at 1731072714993 (+2 ms)Running coprocessor post-open hooks at 1731072715003 (+10 ms)Region opened successfully at 1731072715003 2024-11-08T13:31:55,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=64 2024-11-08T13:31:55,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 00f5b98ab61f3475c8968b8a7a9f6cb4, server=6ccfea20c990,35367,1731072525051 in 496 msec 2024-11-08T13:31:55,003 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57., pid=78, masterSystemTime=1731072714668 2024-11-08T13:31:55,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=00f5b98ab61f3475c8968b8a7a9f6cb4, ASSIGN in 658 msec 2024-11-08T13:31:55,005 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57. 2024-11-08T13:31:55,005 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57. 2024-11-08T13:31:55,006 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe. 2024-11-08T13:31:55,006 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => c4e1216db90b6f396355f2b6bc308ffe, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-08T13:31:55,006 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:31:55,006 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=fcf86fec97b560cab0f86ca4727e4e57, regionState=OPEN, openSeqNum=5, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:55,006 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:31:55,006 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:31:55,006 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:31:55,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure fcf86fec97b560cab0f86ca4727e4e57, server=6ccfea20c990,35061,1731072525012 because future has completed 2024-11-08T13:31:55,008 INFO [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:31:55,010 INFO [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4e1216db90b6f396355f2b6bc308ffe columnFamilyName colfamily11 2024-11-08T13:31:55,010 DEBUG [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:55,010 INFO [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] regionserver.HStore(327): Store=c4e1216db90b6f396355f2b6bc308ffe/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:55,010 INFO [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:31:55,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=72 2024-11-08T13:31:55,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure fcf86fec97b560cab0f86ca4727e4e57, server=6ccfea20c990,35061,1731072525012 in 480 msec 2024-11-08T13:31:55,012 INFO [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4e1216db90b6f396355f2b6bc308ffe columnFamilyName colfamily21 2024-11-08T13:31:55,012 DEBUG [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:55,012 INFO [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] regionserver.HStore(327): Store=c4e1216db90b6f396355f2b6bc308ffe/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:55,012 INFO [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:31:55,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fcf86fec97b560cab0f86ca4727e4e57, ASSIGN in 667 msec 2024-11-08T13:31:55,014 INFO [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c4e1216db90b6f396355f2b6bc308ffe columnFamilyName colfamily31 2024-11-08T13:31:55,014 DEBUG [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:31:55,014 INFO [StoreOpener-c4e1216db90b6f396355f2b6bc308ffe-1 {}] regionserver.HStore(327): Store=c4e1216db90b6f396355f2b6bc308ffe/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:31:55,014 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:31:55,015 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:31:55,016 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:31:55,017 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe/recovered.edits/0000000000000000004 2024-11-08T13:31:55,022 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe/recovered.edits/0000000000000000004 2024-11-08T13:31:55,022 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(2902): Flushing c4e1216db90b6f396355f2b6bc308ffe 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:55,039 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe/.tmp/colfamily11/c25bf335e5324e2a906297b9124acb63 is 52, key is ggg/colfamily11:colfamily11/1731072530779/Put/seqid=0 2024-11-08T13:31:55,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741886_1062 (size=39263) 2024-11-08T13:31:55,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741886_1062 (size=39263) 2024-11-08T13:31:55,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741886_1062 (size=39263) 2024-11-08T13:31:55,047 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe/.tmp/colfamily11/c25bf335e5324e2a906297b9124acb63 2024-11-08T13:31:55,056 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe/.tmp/colfamily11/c25bf335e5324e2a906297b9124acb63 as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe/colfamily11/c25bf335e5324e2a906297b9124acb63 2024-11-08T13:31:55,062 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe/colfamily11/c25bf335e5324e2a906297b9124acb63, entries=703, sequenceid=4, filesize=38.3 K 2024-11-08T13:31:55,063 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for c4e1216db90b6f396355f2b6bc308ffe in 41ms, sequenceid=4, compaction requested=false; wal=null 2024-11-08T13:31:55,064 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe/recovered.edits/0000000000000000004 2024-11-08T13:31:55,065 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:31:55,065 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:31:55,065 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-08T13:31:55,067 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:31:55,069 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T13:31:55,070 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened c4e1216db90b6f396355f2b6bc308ffe; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66927708, jitterRate=-0.002699434757232666}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-08T13:31:55,071 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:31:55,071 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for c4e1216db90b6f396355f2b6bc308ffe: Running coprocessor pre-open hook at 1731072715006Writing region info on filesystem at 1731072715006Initializing all the Stores at 1731072715008 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072715008Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072715008Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072715008Obtaining lock to block concurrent updates at 1731072715022 (+14 ms)Preparing flush snapshotting stores in c4e1216db90b6f396355f2b6bc308ffe at 1731072715022Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072715022Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe. at 1731072715022Flushing c4e1216db90b6f396355f2b6bc308ffe/colfamily11: creating writer at 1731072715022Flushing c4e1216db90b6f396355f2b6bc308ffe/colfamily11: appending metadata at 1731072715038 (+16 ms)Flushing c4e1216db90b6f396355f2b6bc308ffe/colfamily11: closing flushed file at 1731072715038Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13d0c042: reopening flushed file at 1731072715055 (+17 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for c4e1216db90b6f396355f2b6bc308ffe in 41ms, sequenceid=4, compaction requested=false; wal=null at 1731072715063 (+8 ms)Cleaning up temporary data from old regions at 1731072715065 (+2 ms)Running coprocessor post-open hooks at 1731072715071 (+6 ms)Region opened successfully at 1731072715071 2024-11-08T13:31:55,071 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe., pid=76, masterSystemTime=1731072714668 2024-11-08T13:31:55,073 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe. 2024-11-08T13:31:55,074 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe. 2024-11-08T13:31:55,074 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=c4e1216db90b6f396355f2b6bc308ffe, regionState=OPEN, openSeqNum=5, regionLocation=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:55,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure c4e1216db90b6f396355f2b6bc308ffe, server=6ccfea20c990,35061,1731072525012 because future has completed 2024-11-08T13:31:55,079 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=68 2024-11-08T13:31:55,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure c4e1216db90b6f396355f2b6bc308ffe, server=6ccfea20c990,35061,1731072525012 in 554 msec 2024-11-08T13:31:55,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=57 2024-11-08T13:31:55,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c4e1216db90b6f396355f2b6bc308ffe, ASSIGN in 735 msec 2024-11-08T13:31:55,082 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 6ccfea20c990,39565,1731072524927 after splitting done 2024-11-08T13:31:55,082 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-08T13:31:55,082 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:31:55,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:31:55,083 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T13:31:55,095 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 8 exec state=pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:31:55,095 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-08T13:31:55,095 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:31:55,095 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T13:31:55,095 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T13:31:55,096 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-08T13:31:55,096 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-08T13:31:55,096 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-08T13:31:55,096 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T13:31:55,097 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T13:31:55,099 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-08T13:31:55,099 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T13:31:55,099 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T13:31:55,100 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T13:31:55,100 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T13:31:55,100 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T13:31:55,100 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T13:31:55,100 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T13:31:55,100 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T13:31:55,100 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T13:31:55,100 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T13:31:55,100 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T13:31:55,100 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T13:31:55,100 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T13:31:55,100 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T13:31:55,100 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T13:31:55,100 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T13:31:55,100 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T13:31:55,100 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T13:31:55,100 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T13:31:55,100 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T13:31:55,100 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-08T13:31:55,100 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-08T13:31:55,100 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-08T13:31:55,100 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-08T13:31:55,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T13:31:55,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-08T13:31:55,104 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_FINISH, hasLock=false; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.1731072526271, worker=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.1731072526271, worker=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=00f5b98ab61f3475c8968b8a7a9f6cb4, ASSIGN 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5970d249471986ac22ae194d250f9a38, ASSIGN 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ef0046eead8230ad7213c161e3532bd, ASSIGN 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=99abbd965e76aa0c5fe5a1011e45a9ae, ASSIGN 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c4e1216db90b6f396355f2b6bc308ffe, ASSIGN 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7c8fbd70358d0bf3e3f164569432ecfe, ASSIGN 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4d5935497231430a7057c66d28a9d018, ASSIGN 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06547784f6904a15d1d127f091988ed3, ASSIGN 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fcf86fec97b560cab0f86ca4727e4e57, ASSIGN 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 00f5b98ab61f3475c8968b8a7a9f6cb4, server=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure 99abbd965e76aa0c5fe5a1011e45a9ae, server=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure 4d5935497231430a7057c66d28a9d018, server=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure c4e1216db90b6f396355f2b6bc308ffe, server=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 5ef0046eead8230ad7213c161e3532bd, server=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure fcf86fec97b560cab0f86ca4727e4e57, server=6ccfea20c990,35061,1731072525012 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 7c8fbd70358d0bf3e3f164569432ecfe, server=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure 5970d249471986ac22ae194d250f9a38, server=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure 06547784f6904a15d1d127f091988ed3, server=6ccfea20c990,35367,1731072525051 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_FINISH, hasLock=false; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.meta.1731072526718.meta, worker=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 6ccfea20c990%2C39565%2C1731072524927.1731072526271, worker=6ccfea20c990,35061,1731072525012 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6ccfea20c990%2C39565%2C1731072524927.1731072526271, worker=6ccfea20c990,35061,1731072525012 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=00f5b98ab61f3475c8968b8a7a9f6cb4, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5970d249471986ac22ae194d250f9a38, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,105 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5ef0046eead8230ad7213c161e3532bd, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=99abbd965e76aa0c5fe5a1011e45a9ae, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c4e1216db90b6f396355f2b6bc308ffe, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7c8fbd70358d0bf3e3f164569432ecfe, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4d5935497231430a7057c66d28a9d018, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06547784f6904a15d1d127f091988ed3, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=fcf86fec97b560cab0f86ca4727e4e57, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 00f5b98ab61f3475c8968b8a7a9f6cb4, server=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure 99abbd965e76aa0c5fe5a1011e45a9ae, server=6ccfea20c990,35061,1731072525012 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure 4d5935497231430a7057c66d28a9d018, server=6ccfea20c990,35061,1731072525012 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure c4e1216db90b6f396355f2b6bc308ffe, server=6ccfea20c990,35061,1731072525012 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 5ef0046eead8230ad7213c161e3532bd, server=6ccfea20c990,35061,1731072525012 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure fcf86fec97b560cab0f86ca4727e4e57, server=6ccfea20c990,35061,1731072525012 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 7c8fbd70358d0bf3e3f164569432ecfe, server=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure 5970d249471986ac22ae194d250f9a38, server=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,106 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure 06547784f6904a15d1d127f091988ed3, server=6ccfea20c990,35367,1731072525051 didn't hold the lock before restarting, skip acquiring lock. 2024-11-08T13:31:55,106 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-08T13:31:55,107 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=61 2024-11-08T13:31:55,107 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=6ccfea20c990,35367,1731072525051, table=hbase:meta, region=1588230740 2024-11-08T13:31:55,107 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ccfea20c990,35367,1731072525051, state=OPEN 2024-11-08T13:31:55,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:55,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:55,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:31:55,109 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:55,109 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:55,109 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:31:55,110 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731072745109 2024-11-08T13:31:55,110 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T13:31:55,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c6699738a1be7c8647568269e341753c, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:55,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4d5935497231430a7057c66d28a9d018, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=5 2024-11-08T13:31:55,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7e4837322188444dc48d5c8632c81ce5, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:55,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7c8fbd70358d0bf3e3f164569432ecfe, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=5 2024-11-08T13:31:55,117 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0f0799092106200541bc0c416b0bdbb5, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:55,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d85397130af5b1fc184790336ef5d2b8, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:55,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c4e1216db90b6f396355f2b6bc308ffe, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=5 2024-11-08T13:31:55,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=286367177ffe2bf304003f3b829326ac, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:55,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=50c49cba3ff42064b0e071b4b0d3656e, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:55,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5970d249471986ac22ae194d250f9a38, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=5 2024-11-08T13:31:55,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5503a38fb5ee20ab8673683ba1a1bcaa, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:55,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e178bdc5865404fdcd9e707b6918569c, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:55,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ef0046eead8230ad7213c161e3532bd, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=5 2024-11-08T13:31:55,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b082e035d4bc82dc2f93642d66c000c, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:55,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5ab60b8cfddafd2fcce990e602b31aef, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:55,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06547784f6904a15d1d127f091988ed3, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=5 2024-11-08T13:31:55,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c02aa10a67704d3f15deae347223dc13, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:55,118 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fcf86fec97b560cab0f86ca4727e4e57, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=5 2024-11-08T13:31:55,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=223bf549b03e7990028b7927dd897a23, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:55,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=30f6256cfa21047086ecffea588e9713, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:55,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b7f58040b9958dcdcc78f264a8778203, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:55,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=00f5b98ab61f3475c8968b8a7a9f6cb4, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=5 2024-11-08T13:31:55,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4fbf6f08f931761acf639331e9fd9170, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:55,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f973a563f81a494eeb1443f81f0bb069, regionState=OPEN, lastHost=6ccfea20c990,35367,1731072525051, regionLocation=6ccfea20c990,35367,1731072525051, openSeqNum=2 2024-11-08T13:31:55,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99abbd965e76aa0c5fe5a1011e45a9ae, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=5 2024-11-08T13:31:55,119 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e1c2472f75aa6a0ed9ba5017889bd71b, regionState=OPEN, lastHost=6ccfea20c990,35061,1731072525012, regionLocation=6ccfea20c990,35061,1731072525012, openSeqNum=2 2024-11-08T13:31:55,119 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-08T13:31:55,119 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731072775119 2024-11-08T13:31:55,119 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731072835119 2024-11-08T13:31:55,119 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-08T13:31:55,120 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 6ccfea20c990,39565,1731072524927 after splitting done 2024-11-08T13:31:55,120 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-08T13:31:55,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; ServerCrashProcedure 6ccfea20c990,39565,1731072524927, splitWal=true, meta=true in 3 mins, 3.475 sec 2024-11-08T13:31:55,376 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T13:31:55,379 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43212, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T13:31:55,393 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:39565 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 6ccfea20c990/172.17.0.2:39565 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:31:55,394 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018., hostname=6ccfea20c990,39565,1731072524927, seqNum=2, error=java.net.ConnectException: Call to address=6ccfea20c990:39565 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:31:55,395 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 is java.net.ConnectException: Connection refused 2024-11-08T13:31:55,395 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 from cache 2024-11-08T13:31:55,395 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:39565 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 6ccfea20c990/172.17.0.2:39565 2024-11-08T13:31:55,500 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6ccfea20c990:39565 this server is in the failed servers list 2024-11-08T13:31:55,500 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:31:55,500 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:31:55,500 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,39565,1731072524927, seqNum=-1 from cache 2024-11-08T13:31:55,610 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:31:55,611 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T13:31:55,612 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,35367,1731072525051, seqNum=-1] 2024-11-08T13:31:55,614 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T13:31:55,621 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018., hostname=6ccfea20c990,35061,1731072525012, seqNum=5] 2024-11-08T13:31:55,641 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6ccfea20c990:39565 this server is in the failed servers list 2024-11-08T13:31:55,641 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe., hostname=6ccfea20c990,39565,1731072524927, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:31:55,641 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:31:55,641 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 from cache 2024-11-08T13:31:55,752 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe., hostname=6ccfea20c990,35367,1731072525051, seqNum=5] 2024-11-08T13:31:55,767 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6ccfea20c990:39565 this server is in the failed servers list 2024-11-08T13:31:55,767 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe., hostname=6ccfea20c990,39565,1731072524927, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:31:55,767 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:31:55,767 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 from cache 2024-11-08T13:31:55,874 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe., hostname=6ccfea20c990,35061,1731072525012, seqNum=5] 2024-11-08T13:31:55,887 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6ccfea20c990:39565 this server is in the failed servers list 2024-11-08T13:31:55,888 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38., hostname=6ccfea20c990,39565,1731072524927, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:31:55,888 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:31:55,888 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 from cache 2024-11-08T13:31:55,994 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38., hostname=6ccfea20c990,35367,1731072525051, seqNum=5] 2024-11-08T13:31:56,011 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6ccfea20c990:39565 this server is in the failed servers list 2024-11-08T13:31:56,012 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd., hostname=6ccfea20c990,39565,1731072524927, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:31:56,012 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:31:56,012 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 from cache 2024-11-08T13:31:56,124 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd., hostname=6ccfea20c990,35061,1731072525012, seqNum=5] 2024-11-08T13:31:56,136 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6ccfea20c990:39565 this server is in the failed servers list 2024-11-08T13:31:56,137 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3., hostname=6ccfea20c990,39565,1731072524927, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:31:56,137 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:31:56,137 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 from cache 2024-11-08T13:31:56,245 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3., hostname=6ccfea20c990,35367,1731072525051, seqNum=5] 2024-11-08T13:31:56,256 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6ccfea20c990:39565 this server is in the failed servers list 2024-11-08T13:31:56,256 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57., hostname=6ccfea20c990,39565,1731072524927, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:31:56,256 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:31:56,256 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 from cache 2024-11-08T13:31:56,366 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57., hostname=6ccfea20c990,35061,1731072525012, seqNum=5] 2024-11-08T13:31:56,382 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6ccfea20c990:39565 this server is in the failed servers list 2024-11-08T13:31:56,382 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4., hostname=6ccfea20c990,39565,1731072524927, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:31:56,382 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:31:56,382 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 from cache 2024-11-08T13:31:56,495 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4., hostname=6ccfea20c990,35367,1731072525051, seqNum=5] 2024-11-08T13:31:56,508 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6ccfea20c990:39565 this server is in the failed servers list 2024-11-08T13:31:56,508 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae., hostname=6ccfea20c990,39565,1731072524927, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6ccfea20c990:39565 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:31:56,508 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6ccfea20c990:39565 2024-11-08T13:31:56,508 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae., hostname=6ccfea20c990,39565,1731072524927, seqNum=2 from cache 2024-11-08T13:31:56,626 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae., hostname=6ccfea20c990,35061,1731072525012, seqNum=5] 2024-11-08T13:31:56,699 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-08T13:31:56,699 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T13:31:56,699 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T13:31:56,699 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSCPBase.tearDown(TestSCPBase.java:72) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T13:31:56,699 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:31:56,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:31:56,700 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T13:31:56,700 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T13:31:56,700 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2139241463, stopped=false 2024-11-08T13:31:56,700 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=6ccfea20c990,40009,1731072524365 2024-11-08T13:31:56,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T13:31:56,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T13:31:56,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:31:56,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T13:31:56,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:31:56,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:31:56,701 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:31:56,702 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T13:31:56,702 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSCPBase.tearDown(TestSCPBase.java:72) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T13:31:56,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:31:56,702 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:31:56,702 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:31:56,702 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6ccfea20c990,35061,1731072525012' ***** 2024-11-08T13:31:56,702 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T13:31:56,702 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6ccfea20c990,35367,1731072525051' ***** 2024-11-08T13:31:56,702 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T13:31:56,702 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:31:56,702 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T13:31:56,702 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T13:31:56,703 INFO [RS:1;6ccfea20c990:35061 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T13:31:56,703 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T13:31:56,703 INFO [RS:1;6ccfea20c990:35061 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T13:31:56,703 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(3091): Received CLOSE for 30f6256cfa21047086ecffea588e9713 2024-11-08T13:31:56,703 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T13:31:56,703 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(3091): Received CLOSE for 4d5935497231430a7057c66d28a9d018 2024-11-08T13:31:56,703 INFO [RS:2;6ccfea20c990:35367 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T13:31:56,703 INFO [RS:2;6ccfea20c990:35367 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T13:31:56,703 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(3091): Received CLOSE for 286367177ffe2bf304003f3b829326ac 2024-11-08T13:31:56,703 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(3091): Received CLOSE for b7f58040b9958dcdcc78f264a8778203 2024-11-08T13:31:56,703 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(3091): Received CLOSE for c02aa10a67704d3f15deae347223dc13 2024-11-08T13:31:56,703 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(3091): Received CLOSE for e1c2472f75aa6a0ed9ba5017889bd71b 2024-11-08T13:31:56,703 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(3091): Received CLOSE for e178bdc5865404fdcd9e707b6918569c 2024-11-08T13:31:56,704 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(3091): Received CLOSE for c4e1216db90b6f396355f2b6bc308ffe 2024-11-08T13:31:56,704 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(3091): Received CLOSE for 5503a38fb5ee20ab8673683ba1a1bcaa 2024-11-08T13:31:56,704 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(3091): Received CLOSE for 4fbf6f08f931761acf639331e9fd9170 2024-11-08T13:31:56,704 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(3091): Received CLOSE for 00f5b98ab61f3475c8968b8a7a9f6cb4 2024-11-08T13:31:56,704 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(3091): Received CLOSE for 0f0799092106200541bc0c416b0bdbb5 2024-11-08T13:31:56,704 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(3091): Received CLOSE for 223bf549b03e7990028b7927dd897a23 2024-11-08T13:31:56,704 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(3091): Received CLOSE for c6699738a1be7c8647568269e341753c 2024-11-08T13:31:56,704 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(3091): Received CLOSE for f973a563f81a494eeb1443f81f0bb069 2024-11-08T13:31:56,704 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(3091): Received CLOSE for fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:31:56,704 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(3091): Received CLOSE for 0b082e035d4bc82dc2f93642d66c000c 2024-11-08T13:31:56,704 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(3091): Received CLOSE for 5ef0046eead8230ad7213c161e3532bd 2024-11-08T13:31:56,704 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 30f6256cfa21047086ecffea588e9713, disabling compactions & flushes 2024-11-08T13:31:56,704 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b7f58040b9958dcdcc78f264a8778203, disabling compactions & flushes 2024-11-08T13:31:56,704 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(3091): Received CLOSE for 99abbd965e76aa0c5fe5a1011e45a9ae 2024-11-08T13:31:56,704 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(3091): Received CLOSE for d85397130af5b1fc184790336ef5d2b8 2024-11-08T13:31:56,704 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713. 2024-11-08T13:31:56,704 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203. 2024-11-08T13:31:56,704 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(3091): Received CLOSE for 5ab60b8cfddafd2fcce990e602b31aef 2024-11-08T13:31:56,704 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203. 2024-11-08T13:31:56,704 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(3091): Received CLOSE for 50c49cba3ff42064b0e071b4b0d3656e 2024-11-08T13:31:56,704 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713. 2024-11-08T13:31:56,704 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203. after waiting 0 ms 2024-11-08T13:31:56,704 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713. after waiting 0 ms 2024-11-08T13:31:56,704 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(3091): Received CLOSE for 06547784f6904a15d1d127f091988ed3 2024-11-08T13:31:56,704 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203. 2024-11-08T13:31:56,704 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(959): stopping server 6ccfea20c990,35061,1731072525012 2024-11-08T13:31:56,704 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713. 2024-11-08T13:31:56,704 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(3091): Received CLOSE for 7e4837322188444dc48d5c8632c81ce5 2024-11-08T13:31:56,704 INFO [RS:1;6ccfea20c990:35061 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T13:31:56,704 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(3091): Received CLOSE for 5970d249471986ac22ae194d250f9a38 2024-11-08T13:31:56,704 INFO [RS:1;6ccfea20c990:35061 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;6ccfea20c990:35061. 2024-11-08T13:31:56,704 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(3091): Received CLOSE for 7c8fbd70358d0bf3e3f164569432ecfe 2024-11-08T13:31:56,704 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing b7f58040b9958dcdcc78f264a8778203 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:56,704 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(959): stopping server 6ccfea20c990,35367,1731072525051 2024-11-08T13:31:56,704 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 30f6256cfa21047086ecffea588e9713 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:56,704 INFO [RS:2;6ccfea20c990:35367 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T13:31:56,704 DEBUG [RS:1;6ccfea20c990:35061 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T13:31:56,704 DEBUG [RS:1;6ccfea20c990:35061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:31:56,704 INFO [RS:2;6ccfea20c990:35367 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;6ccfea20c990:35367. 2024-11-08T13:31:56,704 DEBUG [RS:2;6ccfea20c990:35367 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T13:31:56,704 DEBUG [RS:2;6ccfea20c990:35367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:31:56,704 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-08T13:31:56,705 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T13:31:56,705 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T13:31:56,705 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T13:31:56,705 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T13:31:56,705 DEBUG [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(1351): Waiting on 0f0799092106200541bc0c416b0bdbb5, 286367177ffe2bf304003f3b829326ac, 30f6256cfa21047086ecffea588e9713, 4d5935497231430a7057c66d28a9d018, 4fbf6f08f931761acf639331e9fd9170, 5ab60b8cfddafd2fcce990e602b31aef, 5ef0046eead8230ad7213c161e3532bd, 99abbd965e76aa0c5fe5a1011e45a9ae, c02aa10a67704d3f15deae347223dc13, c4e1216db90b6f396355f2b6bc308ffe, c6699738a1be7c8647568269e341753c, e178bdc5865404fdcd9e707b6918569c, e1c2472f75aa6a0ed9ba5017889bd71b, fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:31:56,705 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-08T13:31:56,705 DEBUG [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(1351): Waiting on 00f5b98ab61f3475c8968b8a7a9f6cb4, 06547784f6904a15d1d127f091988ed3, 0b082e035d4bc82dc2f93642d66c000c, 1588230740, 223bf549b03e7990028b7927dd897a23, 50c49cba3ff42064b0e071b4b0d3656e, 5503a38fb5ee20ab8673683ba1a1bcaa, 5970d249471986ac22ae194d250f9a38, 7c8fbd70358d0bf3e3f164569432ecfe, 7e4837322188444dc48d5c8632c81ce5, b7f58040b9958dcdcc78f264a8778203, d85397130af5b1fc184790336ef5d2b8, f973a563f81a494eeb1443f81f0bb069 2024-11-08T13:31:56,705 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T13:31:56,705 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T13:31:56,705 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T13:31:56,705 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T13:31:56,705 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T13:31:56,705 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=13.39 KB heapSize=19.70 KB 2024-11-08T13:31:56,715 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/.tmp/info/73cc6a4b76fd4a2cbde5f8df59f75da3 is 268, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018./info:regioninfo/1731072714919/Put/seqid=0 2024-11-08T13:31:56,726 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/30f6256cfa21047086ecffea588e9713/.tmp/colfamily11/d299c4bcef2f43c697256036f0d45b0e is 52, key is ttt/colfamily11:colfamily11/1731072530922/Put/seqid=0 2024-11-08T13:31:56,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741887_1063 (size=19858) 2024-11-08T13:31:56,729 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b7f58040b9958dcdcc78f264a8778203/.tmp/colfamily11/13f5cb700a0f40c8a4ea84221af1fdff is 52, key is uuu/colfamily11:colfamily11/1731072530944/Put/seqid=0 2024-11-08T13:31:56,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741887_1063 (size=19858) 2024-11-08T13:31:56,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741887_1063 (size=19858) 2024-11-08T13:31:56,730 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.39 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/.tmp/info/73cc6a4b76fd4a2cbde5f8df59f75da3 2024-11-08T13:31:56,738 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/.tmp/info/73cc6a4b76fd4a2cbde5f8df59f75da3 as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/info/73cc6a4b76fd4a2cbde5f8df59f75da3 2024-11-08T13:31:56,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741889_1065 (size=39263) 2024-11-08T13:31:56,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741889_1065 (size=39263) 2024-11-08T13:31:56,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741889_1065 (size=39263) 2024-11-08T13:31:56,744 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b7f58040b9958dcdcc78f264a8778203/.tmp/colfamily11/13f5cb700a0f40c8a4ea84221af1fdff 2024-11-08T13:31:56,746 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/info/73cc6a4b76fd4a2cbde5f8df59f75da3, entries=72, sequenceid=81, filesize=19.4 K 2024-11-08T13:31:56,747 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~13.39 KB/13716, heapSize ~18.94 KB/19392, currentSize=0 B/0 for 1588230740 in 42ms, sequenceid=81, compaction requested=false 2024-11-08T13:31:56,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741888_1064 (size=39263) 2024-11-08T13:31:56,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741888_1064 (size=39263) 2024-11-08T13:31:56,752 INFO [regionserver/6ccfea20c990:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T13:31:56,753 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b7f58040b9958dcdcc78f264a8778203/.tmp/colfamily11/13f5cb700a0f40c8a4ea84221af1fdff as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b7f58040b9958dcdcc78f264a8778203/colfamily11/13f5cb700a0f40c8a4ea84221af1fdff 2024-11-08T13:31:56,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741888_1064 (size=39263) 2024-11-08T13:31:56,753 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/30f6256cfa21047086ecffea588e9713/.tmp/colfamily11/d299c4bcef2f43c697256036f0d45b0e 2024-11-08T13:31:56,755 INFO [regionserver/6ccfea20c990:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T13:31:56,759 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b7f58040b9958dcdcc78f264a8778203/colfamily11/13f5cb700a0f40c8a4ea84221af1fdff, entries=703, sequenceid=5, filesize=38.3 K 2024-11-08T13:31:56,760 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for b7f58040b9958dcdcc78f264a8778203 in 56ms, sequenceid=5, compaction requested=false 2024-11-08T13:31:56,760 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/30f6256cfa21047086ecffea588e9713/.tmp/colfamily11/d299c4bcef2f43c697256036f0d45b0e as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/30f6256cfa21047086ecffea588e9713/colfamily11/d299c4bcef2f43c697256036f0d45b0e 2024-11-08T13:31:56,765 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b7f58040b9958dcdcc78f264a8778203/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-08T13:31:56,768 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/30f6256cfa21047086ecffea588e9713/colfamily11/d299c4bcef2f43c697256036f0d45b0e, entries=703, sequenceid=5, filesize=38.3 K 2024-11-08T13:31:56,769 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 30f6256cfa21047086ecffea588e9713 in 65ms, sequenceid=5, compaction requested=false 2024-11-08T13:31:56,769 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203. 2024-11-08T13:31:56,769 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b7f58040b9958dcdcc78f264a8778203: Waiting for close lock at 1731072716704Running coprocessor pre-close hooks at 1731072716704Disabling compacts and flushes for region at 1731072716704Disabling writes for close at 1731072716704Obtaining lock to block concurrent updates at 1731072716704Preparing flush snapshotting stores in b7f58040b9958dcdcc78f264a8778203 at 1731072716704Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072716708 (+4 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203. at 1731072716710 (+2 ms)Flushing b7f58040b9958dcdcc78f264a8778203/colfamily11: creating writer at 1731072716710Flushing b7f58040b9958dcdcc78f264a8778203/colfamily11: appending metadata at 1731072716728 (+18 ms)Flushing b7f58040b9958dcdcc78f264a8778203/colfamily11: closing flushed file at 1731072716729 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18265918: reopening flushed file at 1731072716752 (+23 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for b7f58040b9958dcdcc78f264a8778203 in 56ms, sequenceid=5, compaction requested=false at 1731072716761 (+9 ms)Writing region close event to WAL at 1731072716761Running coprocessor post-close hooks at 1731072716766 (+5 ms)Closed at 1731072716769 (+3 ms) 2024-11-08T13:31:56,770 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/hbase/meta/1588230740/recovered.edits/84.seqid, newMaxSeqId=84, maxSeqId=60 2024-11-08T13:31:56,770 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731072527108.b7f58040b9958dcdcc78f264a8778203. 2024-11-08T13:31:56,770 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5503a38fb5ee20ab8673683ba1a1bcaa, disabling compactions & flushes 2024-11-08T13:31:56,770 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa. 2024-11-08T13:31:56,770 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa. 2024-11-08T13:31:56,770 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa. after waiting 0 ms 2024-11-08T13:31:56,770 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa. 2024-11-08T13:31:56,770 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 5503a38fb5ee20ab8673683ba1a1bcaa 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:56,771 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T13:31:56,771 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T13:31:56,771 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731072716705Running coprocessor pre-close hooks at 1731072716705Disabling compacts and flushes for region at 1731072716705Disabling writes for close at 1731072716705Obtaining lock to block concurrent updates at 1731072716705Preparing flush snapshotting stores in 1588230740 at 1731072716705Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=13716, getHeapSize=20112, getOffHeapSize=0, getCellsCount=72 at 1731072716708 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731072716709 (+1 ms)Flushing 1588230740/info: creating writer at 1731072716709Flushing 1588230740/info: appending metadata at 1731072716714 (+5 ms)Flushing 1588230740/info: closing flushed file at 1731072716714Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2231d3c0: reopening flushed file at 1731072716737 (+23 ms)Finished flush of dataSize ~13.39 KB/13716, heapSize ~18.94 KB/19392, currentSize=0 B/0 for 1588230740 in 42ms, sequenceid=81, compaction requested=false at 1731072716747 (+10 ms)Writing region close event to WAL at 1731072716766 (+19 ms)Running coprocessor post-close hooks at 1731072716771 (+5 ms)Closed at 1731072716771 2024-11-08T13:31:56,771 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T13:31:56,775 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/30f6256cfa21047086ecffea588e9713/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-08T13:31:56,776 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713. 2024-11-08T13:31:56,776 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 30f6256cfa21047086ecffea588e9713: Waiting for close lock at 1731072716703Running coprocessor pre-close hooks at 1731072716704 (+1 ms)Disabling compacts and flushes for region at 1731072716704Disabling writes for close at 1731072716704Obtaining lock to block concurrent updates at 1731072716704Preparing flush snapshotting stores in 30f6256cfa21047086ecffea588e9713 at 1731072716704Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072716708 (+4 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713. at 1731072716709 (+1 ms)Flushing 30f6256cfa21047086ecffea588e9713/colfamily11: creating writer at 1731072716709Flushing 30f6256cfa21047086ecffea588e9713/colfamily11: appending metadata at 1731072716726 (+17 ms)Flushing 30f6256cfa21047086ecffea588e9713/colfamily11: closing flushed file at 1731072716726Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51031d78: reopening flushed file at 1731072716759 (+33 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 30f6256cfa21047086ecffea588e9713 in 65ms, sequenceid=5, compaction requested=false at 1731072716769 (+10 ms)Writing region close event to WAL at 1731072716771 (+2 ms)Running coprocessor post-close hooks at 1731072716776 (+5 ms)Closed at 1731072716776 2024-11-08T13:31:56,777 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731072527108.30f6256cfa21047086ecffea588e9713. 2024-11-08T13:31:56,777 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4d5935497231430a7057c66d28a9d018, disabling compactions & flushes 2024-11-08T13:31:56,777 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018. 2024-11-08T13:31:56,777 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018. 2024-11-08T13:31:56,777 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018. after waiting 0 ms 2024-11-08T13:31:56,777 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018. 2024-11-08T13:31:56,781 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4d5935497231430a7057c66d28a9d018/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-08T13:31:56,782 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018. 2024-11-08T13:31:56,782 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4d5935497231430a7057c66d28a9d018: Waiting for close lock at 1731072716777Running coprocessor pre-close hooks at 1731072716777Disabling compacts and flushes for region at 1731072716777Disabling writes for close at 1731072716777Writing region close event to WAL at 1731072716778 (+1 ms)Running coprocessor post-close hooks at 1731072716782 (+4 ms)Closed at 1731072716782 2024-11-08T13:31:56,782 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731072527108.4d5935497231430a7057c66d28a9d018. 2024-11-08T13:31:56,782 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 286367177ffe2bf304003f3b829326ac, disabling compactions & flushes 2024-11-08T13:31:56,782 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac. 2024-11-08T13:31:56,782 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac. 2024-11-08T13:31:56,782 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac. after waiting 0 ms 2024-11-08T13:31:56,782 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac. 2024-11-08T13:31:56,782 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 286367177ffe2bf304003f3b829326ac 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:56,789 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5503a38fb5ee20ab8673683ba1a1bcaa/.tmp/colfamily11/8ea1565e3a0a45e9942b3f58d4d369b4 is 52, key is kkk/colfamily11:colfamily11/1731072530830/Put/seqid=0 2024-11-08T13:31:56,799 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/286367177ffe2bf304003f3b829326ac/.tmp/colfamily11/115c8686a7924c969e52ef77ca0981ce is 52, key is hhh/colfamily11:colfamily11/1731072530774/Put/seqid=0 2024-11-08T13:31:56,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741890_1066 (size=39263) 2024-11-08T13:31:56,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741890_1066 (size=39263) 2024-11-08T13:31:56,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741890_1066 (size=39263) 2024-11-08T13:31:56,802 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5503a38fb5ee20ab8673683ba1a1bcaa/.tmp/colfamily11/8ea1565e3a0a45e9942b3f58d4d369b4 2024-11-08T13:31:56,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741891_1067 (size=39263) 2024-11-08T13:31:56,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741891_1067 (size=39263) 2024-11-08T13:31:56,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741891_1067 (size=39263) 2024-11-08T13:31:56,812 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5503a38fb5ee20ab8673683ba1a1bcaa/.tmp/colfamily11/8ea1565e3a0a45e9942b3f58d4d369b4 as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5503a38fb5ee20ab8673683ba1a1bcaa/colfamily11/8ea1565e3a0a45e9942b3f58d4d369b4 2024-11-08T13:31:56,820 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5503a38fb5ee20ab8673683ba1a1bcaa/colfamily11/8ea1565e3a0a45e9942b3f58d4d369b4, entries=703, sequenceid=5, filesize=38.3 K 2024-11-08T13:31:56,821 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5503a38fb5ee20ab8673683ba1a1bcaa in 51ms, sequenceid=5, compaction requested=false 2024-11-08T13:31:56,826 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5503a38fb5ee20ab8673683ba1a1bcaa/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-08T13:31:56,826 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa. 2024-11-08T13:31:56,826 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5503a38fb5ee20ab8673683ba1a1bcaa: Waiting for close lock at 1731072716770Running coprocessor pre-close hooks at 1731072716770Disabling compacts and flushes for region at 1731072716770Disabling writes for close at 1731072716770Obtaining lock to block concurrent updates at 1731072716770Preparing flush snapshotting stores in 5503a38fb5ee20ab8673683ba1a1bcaa at 1731072716770Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072716771 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa. at 1731072716772 (+1 ms)Flushing 5503a38fb5ee20ab8673683ba1a1bcaa/colfamily11: creating writer at 1731072716772Flushing 5503a38fb5ee20ab8673683ba1a1bcaa/colfamily11: appending metadata at 1731072716789 (+17 ms)Flushing 5503a38fb5ee20ab8673683ba1a1bcaa/colfamily11: closing flushed file at 1731072716789Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20b22ae1: reopening flushed file at 1731072716811 (+22 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5503a38fb5ee20ab8673683ba1a1bcaa in 51ms, sequenceid=5, compaction requested=false at 1731072716821 (+10 ms)Writing region close event to WAL at 1731072716822 (+1 ms)Running coprocessor post-close hooks at 1731072716826 (+4 ms)Closed at 1731072716826 2024-11-08T13:31:56,826 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731072527108.5503a38fb5ee20ab8673683ba1a1bcaa. 2024-11-08T13:31:56,827 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 00f5b98ab61f3475c8968b8a7a9f6cb4, disabling compactions & flushes 2024-11-08T13:31:56,827 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4. 2024-11-08T13:31:56,827 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4. 2024-11-08T13:31:56,827 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4. after waiting 0 ms 2024-11-08T13:31:56,827 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4. 2024-11-08T13:31:56,831 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/00f5b98ab61f3475c8968b8a7a9f6cb4/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-08T13:31:56,832 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4. 2024-11-08T13:31:56,832 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 00f5b98ab61f3475c8968b8a7a9f6cb4: Waiting for close lock at 1731072716827Running coprocessor pre-close hooks at 1731072716827Disabling compacts and flushes for region at 1731072716827Disabling writes for close at 1731072716827Writing region close event to WAL at 1731072716828 (+1 ms)Running coprocessor post-close hooks at 1731072716832 (+4 ms)Closed at 1731072716832 2024-11-08T13:31:56,832 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731072527108.00f5b98ab61f3475c8968b8a7a9f6cb4. 2024-11-08T13:31:56,832 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 223bf549b03e7990028b7927dd897a23, disabling compactions & flushes 2024-11-08T13:31:56,832 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23. 2024-11-08T13:31:56,832 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23. 2024-11-08T13:31:56,832 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23. after waiting 0 ms 2024-11-08T13:31:56,832 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23. 2024-11-08T13:31:56,833 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 223bf549b03e7990028b7927dd897a23 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:56,853 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/223bf549b03e7990028b7927dd897a23/.tmp/colfamily11/6952b17b30dd40ca83a7169be09017ad is 52, key is sss/colfamily11:colfamily11/1731072530905/Put/seqid=0 2024-11-08T13:31:56,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741892_1068 (size=39263) 2024-11-08T13:31:56,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741892_1068 (size=39263) 2024-11-08T13:31:56,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741892_1068 (size=39263) 2024-11-08T13:31:56,860 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/223bf549b03e7990028b7927dd897a23/.tmp/colfamily11/6952b17b30dd40ca83a7169be09017ad 2024-11-08T13:31:56,866 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/223bf549b03e7990028b7927dd897a23/.tmp/colfamily11/6952b17b30dd40ca83a7169be09017ad as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/223bf549b03e7990028b7927dd897a23/colfamily11/6952b17b30dd40ca83a7169be09017ad 2024-11-08T13:31:56,873 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/223bf549b03e7990028b7927dd897a23/colfamily11/6952b17b30dd40ca83a7169be09017ad, entries=703, sequenceid=5, filesize=38.3 K 2024-11-08T13:31:56,874 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 223bf549b03e7990028b7927dd897a23 in 42ms, sequenceid=5, compaction requested=false 2024-11-08T13:31:56,879 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/223bf549b03e7990028b7927dd897a23/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-08T13:31:56,880 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23. 2024-11-08T13:31:56,880 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 223bf549b03e7990028b7927dd897a23: Waiting for close lock at 1731072716832Running coprocessor pre-close hooks at 1731072716832Disabling compacts and flushes for region at 1731072716832Disabling writes for close at 1731072716832Obtaining lock to block concurrent updates at 1731072716833 (+1 ms)Preparing flush snapshotting stores in 223bf549b03e7990028b7927dd897a23 at 1731072716833Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072716833Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23. at 1731072716833Flushing 223bf549b03e7990028b7927dd897a23/colfamily11: creating writer at 1731072716833Flushing 223bf549b03e7990028b7927dd897a23/colfamily11: appending metadata at 1731072716853 (+20 ms)Flushing 223bf549b03e7990028b7927dd897a23/colfamily11: closing flushed file at 1731072716853Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@249c66f9: reopening flushed file at 1731072716865 (+12 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 223bf549b03e7990028b7927dd897a23 in 42ms, sequenceid=5, compaction requested=false at 1731072716874 (+9 ms)Writing region close event to WAL at 1731072716875 (+1 ms)Running coprocessor post-close hooks at 1731072716880 (+5 ms)Closed at 1731072716880 2024-11-08T13:31:56,880 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731072527108.223bf549b03e7990028b7927dd897a23. 2024-11-08T13:31:56,880 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f973a563f81a494eeb1443f81f0bb069, disabling compactions & flushes 2024-11-08T13:31:56,880 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069. 2024-11-08T13:31:56,880 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069. 2024-11-08T13:31:56,880 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069. after waiting 0 ms 2024-11-08T13:31:56,880 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069. 2024-11-08T13:31:56,880 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f973a563f81a494eeb1443f81f0bb069 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:56,895 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f973a563f81a494eeb1443f81f0bb069/.tmp/colfamily11/fd0be74832034d64be289ec705bfcc3d is 52, key is xxx/colfamily11:colfamily11/1731072530968/Put/seqid=0 2024-11-08T13:31:56,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741893_1069 (size=39263) 2024-11-08T13:31:56,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741893_1069 (size=39263) 2024-11-08T13:31:56,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741893_1069 (size=39263) 2024-11-08T13:31:56,902 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f973a563f81a494eeb1443f81f0bb069/.tmp/colfamily11/fd0be74832034d64be289ec705bfcc3d 2024-11-08T13:31:56,905 DEBUG [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(1351): Waiting on 0f0799092106200541bc0c416b0bdbb5, 286367177ffe2bf304003f3b829326ac, 4fbf6f08f931761acf639331e9fd9170, 5ab60b8cfddafd2fcce990e602b31aef, 5ef0046eead8230ad7213c161e3532bd, 99abbd965e76aa0c5fe5a1011e45a9ae, c02aa10a67704d3f15deae347223dc13, c4e1216db90b6f396355f2b6bc308ffe, c6699738a1be7c8647568269e341753c, e178bdc5865404fdcd9e707b6918569c, e1c2472f75aa6a0ed9ba5017889bd71b, fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:31:56,905 DEBUG [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(1351): Waiting on 06547784f6904a15d1d127f091988ed3, 0b082e035d4bc82dc2f93642d66c000c, 50c49cba3ff42064b0e071b4b0d3656e, 5970d249471986ac22ae194d250f9a38, 7c8fbd70358d0bf3e3f164569432ecfe, 7e4837322188444dc48d5c8632c81ce5, d85397130af5b1fc184790336ef5d2b8, f973a563f81a494eeb1443f81f0bb069 2024-11-08T13:31:56,909 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f973a563f81a494eeb1443f81f0bb069/.tmp/colfamily11/fd0be74832034d64be289ec705bfcc3d as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f973a563f81a494eeb1443f81f0bb069/colfamily11/fd0be74832034d64be289ec705bfcc3d 2024-11-08T13:31:56,915 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f973a563f81a494eeb1443f81f0bb069/colfamily11/fd0be74832034d64be289ec705bfcc3d, entries=703, sequenceid=5, filesize=38.3 K 2024-11-08T13:31:56,916 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for f973a563f81a494eeb1443f81f0bb069 in 36ms, sequenceid=5, compaction requested=false 2024-11-08T13:31:56,921 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f973a563f81a494eeb1443f81f0bb069/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-08T13:31:56,921 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069. 2024-11-08T13:31:56,921 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f973a563f81a494eeb1443f81f0bb069: Waiting for close lock at 1731072716880Running coprocessor pre-close hooks at 1731072716880Disabling compacts and flushes for region at 1731072716880Disabling writes for close at 1731072716880Obtaining lock to block concurrent updates at 1731072716880Preparing flush snapshotting stores in f973a563f81a494eeb1443f81f0bb069 at 1731072716880Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072716881 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069. at 1731072716881Flushing f973a563f81a494eeb1443f81f0bb069/colfamily11: creating writer at 1731072716881Flushing f973a563f81a494eeb1443f81f0bb069/colfamily11: appending metadata at 1731072716895 (+14 ms)Flushing f973a563f81a494eeb1443f81f0bb069/colfamily11: closing flushed file at 1731072716895Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@749d35b7: reopening flushed file at 1731072716908 (+13 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for f973a563f81a494eeb1443f81f0bb069 in 36ms, sequenceid=5, compaction requested=false at 1731072716916 (+8 ms)Writing region close event to WAL at 1731072716917 (+1 ms)Running coprocessor post-close hooks at 1731072716921 (+4 ms)Closed at 1731072716921 2024-11-08T13:31:56,921 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731072527108.f973a563f81a494eeb1443f81f0bb069. 2024-11-08T13:31:56,922 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0b082e035d4bc82dc2f93642d66c000c, disabling compactions & flushes 2024-11-08T13:31:56,922 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c. 2024-11-08T13:31:56,922 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c. 2024-11-08T13:31:56,922 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c. after waiting 0 ms 2024-11-08T13:31:56,922 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c. 2024-11-08T13:31:56,922 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0b082e035d4bc82dc2f93642d66c000c 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:56,937 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0b082e035d4bc82dc2f93642d66c000c/.tmp/colfamily11/fafa0cdccf0b4976ad88c738573e5182 is 52, key is nnn/colfamily11:colfamily11/1731072530867/Put/seqid=0 2024-11-08T13:31:56,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741894_1070 (size=39263) 2024-11-08T13:31:56,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741894_1070 (size=39263) 2024-11-08T13:31:56,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741894_1070 (size=39263) 2024-11-08T13:31:56,944 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0b082e035d4bc82dc2f93642d66c000c/.tmp/colfamily11/fafa0cdccf0b4976ad88c738573e5182 2024-11-08T13:31:56,950 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0b082e035d4bc82dc2f93642d66c000c/.tmp/colfamily11/fafa0cdccf0b4976ad88c738573e5182 as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0b082e035d4bc82dc2f93642d66c000c/colfamily11/fafa0cdccf0b4976ad88c738573e5182 2024-11-08T13:31:56,956 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0b082e035d4bc82dc2f93642d66c000c/colfamily11/fafa0cdccf0b4976ad88c738573e5182, entries=703, sequenceid=5, filesize=38.3 K 2024-11-08T13:31:56,957 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 0b082e035d4bc82dc2f93642d66c000c in 35ms, sequenceid=5, compaction requested=false 2024-11-08T13:31:56,965 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0b082e035d4bc82dc2f93642d66c000c/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-08T13:31:56,965 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c. 2024-11-08T13:31:56,965 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0b082e035d4bc82dc2f93642d66c000c: Waiting for close lock at 1731072716922Running coprocessor pre-close hooks at 1731072716922Disabling compacts and flushes for region at 1731072716922Disabling writes for close at 1731072716922Obtaining lock to block concurrent updates at 1731072716922Preparing flush snapshotting stores in 0b082e035d4bc82dc2f93642d66c000c at 1731072716922Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072716922Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c. at 1731072716923 (+1 ms)Flushing 0b082e035d4bc82dc2f93642d66c000c/colfamily11: creating writer at 1731072716923Flushing 0b082e035d4bc82dc2f93642d66c000c/colfamily11: appending metadata at 1731072716937 (+14 ms)Flushing 0b082e035d4bc82dc2f93642d66c000c/colfamily11: closing flushed file at 1731072716937Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61ad5f32: reopening flushed file at 1731072716949 (+12 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 0b082e035d4bc82dc2f93642d66c000c in 35ms, sequenceid=5, compaction requested=false at 1731072716958 (+9 ms)Writing region close event to WAL at 1731072716961 (+3 ms)Running coprocessor post-close hooks at 1731072716965 (+4 ms)Closed at 1731072716965 2024-11-08T13:31:56,965 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731072527108.0b082e035d4bc82dc2f93642d66c000c. 2024-11-08T13:31:56,966 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d85397130af5b1fc184790336ef5d2b8, disabling compactions & flushes 2024-11-08T13:31:56,966 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8. 2024-11-08T13:31:56,966 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8. 2024-11-08T13:31:56,966 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8. after waiting 0 ms 2024-11-08T13:31:56,966 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8. 2024-11-08T13:31:56,966 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d85397130af5b1fc184790336ef5d2b8 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:56,981 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d85397130af5b1fc184790336ef5d2b8/.tmp/colfamily11/c73147fd4ce049638e35275c86cd1bc8 is 52, key is fff/colfamily11:colfamily11/1731072530714/Put/seqid=0 2024-11-08T13:31:56,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741895_1071 (size=39263) 2024-11-08T13:31:56,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741895_1071 (size=39263) 2024-11-08T13:31:56,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741895_1071 (size=39263) 2024-11-08T13:31:56,988 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d85397130af5b1fc184790336ef5d2b8/.tmp/colfamily11/c73147fd4ce049638e35275c86cd1bc8 2024-11-08T13:31:56,994 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d85397130af5b1fc184790336ef5d2b8/.tmp/colfamily11/c73147fd4ce049638e35275c86cd1bc8 as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d85397130af5b1fc184790336ef5d2b8/colfamily11/c73147fd4ce049638e35275c86cd1bc8 2024-11-08T13:31:56,999 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d85397130af5b1fc184790336ef5d2b8/colfamily11/c73147fd4ce049638e35275c86cd1bc8, entries=703, sequenceid=5, filesize=38.3 K 2024-11-08T13:31:57,000 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for d85397130af5b1fc184790336ef5d2b8 in 33ms, sequenceid=5, compaction requested=false 2024-11-08T13:31:57,004 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d85397130af5b1fc184790336ef5d2b8/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-08T13:31:57,004 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8. 2024-11-08T13:31:57,005 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d85397130af5b1fc184790336ef5d2b8: Waiting for close lock at 1731072716965Running coprocessor pre-close hooks at 1731072716965Disabling compacts and flushes for region at 1731072716965Disabling writes for close at 1731072716966 (+1 ms)Obtaining lock to block concurrent updates at 1731072716966Preparing flush snapshotting stores in d85397130af5b1fc184790336ef5d2b8 at 1731072716966Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072716966Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8. at 1731072716967 (+1 ms)Flushing d85397130af5b1fc184790336ef5d2b8/colfamily11: creating writer at 1731072716967Flushing d85397130af5b1fc184790336ef5d2b8/colfamily11: appending metadata at 1731072716981 (+14 ms)Flushing d85397130af5b1fc184790336ef5d2b8/colfamily11: closing flushed file at 1731072716981Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b82f61d: reopening flushed file at 1731072716993 (+12 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for d85397130af5b1fc184790336ef5d2b8 in 33ms, sequenceid=5, compaction requested=false at 1731072717000 (+7 ms)Writing region close event to WAL at 1731072717000Running coprocessor post-close hooks at 1731072717004 (+4 ms)Closed at 1731072717004 2024-11-08T13:31:57,005 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731072527108.d85397130af5b1fc184790336ef5d2b8. 2024-11-08T13:31:57,005 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 50c49cba3ff42064b0e071b4b0d3656e, disabling compactions & flushes 2024-11-08T13:31:57,005 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e. 2024-11-08T13:31:57,005 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e. 2024-11-08T13:31:57,005 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e. after waiting 0 ms 2024-11-08T13:31:57,005 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e. 2024-11-08T13:31:57,005 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 50c49cba3ff42064b0e071b4b0d3656e 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:57,022 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/50c49cba3ff42064b0e071b4b0d3656e/.tmp/colfamily11/861e160606184bae9f5811233288647a is 52, key is iii/colfamily11:colfamily11/1731072530793/Put/seqid=0 2024-11-08T13:31:57,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741896_1072 (size=39263) 2024-11-08T13:31:57,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741896_1072 (size=39263) 2024-11-08T13:31:57,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741896_1072 (size=39263) 2024-11-08T13:31:57,028 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/50c49cba3ff42064b0e071b4b0d3656e/.tmp/colfamily11/861e160606184bae9f5811233288647a 2024-11-08T13:31:57,034 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/50c49cba3ff42064b0e071b4b0d3656e/.tmp/colfamily11/861e160606184bae9f5811233288647a as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/50c49cba3ff42064b0e071b4b0d3656e/colfamily11/861e160606184bae9f5811233288647a 2024-11-08T13:31:57,040 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/50c49cba3ff42064b0e071b4b0d3656e/colfamily11/861e160606184bae9f5811233288647a, entries=703, sequenceid=5, filesize=38.3 K 2024-11-08T13:31:57,041 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 50c49cba3ff42064b0e071b4b0d3656e in 36ms, sequenceid=5, compaction requested=false 2024-11-08T13:31:57,046 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/50c49cba3ff42064b0e071b4b0d3656e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-08T13:31:57,046 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e. 2024-11-08T13:31:57,046 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 50c49cba3ff42064b0e071b4b0d3656e: Waiting for close lock at 1731072717005Running coprocessor pre-close hooks at 1731072717005Disabling compacts and flushes for region at 1731072717005Disabling writes for close at 1731072717005Obtaining lock to block concurrent updates at 1731072717005Preparing flush snapshotting stores in 50c49cba3ff42064b0e071b4b0d3656e at 1731072717005Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072717006 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e. at 1731072717006Flushing 50c49cba3ff42064b0e071b4b0d3656e/colfamily11: creating writer at 1731072717006Flushing 50c49cba3ff42064b0e071b4b0d3656e/colfamily11: appending metadata at 1731072717022 (+16 ms)Flushing 50c49cba3ff42064b0e071b4b0d3656e/colfamily11: closing flushed file at 1731072717022Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14069d41: reopening flushed file at 1731072717033 (+11 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 50c49cba3ff42064b0e071b4b0d3656e in 36ms, sequenceid=5, compaction requested=false at 1731072717041 (+8 ms)Writing region close event to WAL at 1731072717042 (+1 ms)Running coprocessor post-close hooks at 1731072717046 (+4 ms)Closed at 1731072717046 2024-11-08T13:31:57,046 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731072527108.50c49cba3ff42064b0e071b4b0d3656e. 2024-11-08T13:31:57,047 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 06547784f6904a15d1d127f091988ed3, disabling compactions & flushes 2024-11-08T13:31:57,047 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3. 2024-11-08T13:31:57,047 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3. 2024-11-08T13:31:57,047 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3. after waiting 0 ms 2024-11-08T13:31:57,047 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3. 2024-11-08T13:31:57,051 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06547784f6904a15d1d127f091988ed3/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-08T13:31:57,052 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3. 2024-11-08T13:31:57,052 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 06547784f6904a15d1d127f091988ed3: Waiting for close lock at 1731072717047Running coprocessor pre-close hooks at 1731072717047Disabling compacts and flushes for region at 1731072717047Disabling writes for close at 1731072717047Writing region close event to WAL at 1731072717047Running coprocessor post-close hooks at 1731072717052 (+5 ms)Closed at 1731072717052 2024-11-08T13:31:57,052 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731072527108.06547784f6904a15d1d127f091988ed3. 2024-11-08T13:31:57,052 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7e4837322188444dc48d5c8632c81ce5, disabling compactions & flushes 2024-11-08T13:31:57,052 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5. 2024-11-08T13:31:57,052 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5. 2024-11-08T13:31:57,052 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5. after waiting 0 ms 2024-11-08T13:31:57,052 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5. 2024-11-08T13:31:57,053 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 7e4837322188444dc48d5c8632c81ce5 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:57,069 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7e4837322188444dc48d5c8632c81ce5/.tmp/colfamily11/cf6d9387abc845c69d3b2980cf4c7076 is 52, key is ccc/colfamily11:colfamily11/1731072530666/Put/seqid=0 2024-11-08T13:31:57,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741897_1073 (size=39263) 2024-11-08T13:31:57,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741897_1073 (size=39263) 2024-11-08T13:31:57,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741897_1073 (size=39263) 2024-11-08T13:31:57,075 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7e4837322188444dc48d5c8632c81ce5/.tmp/colfamily11/cf6d9387abc845c69d3b2980cf4c7076 2024-11-08T13:31:57,081 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7e4837322188444dc48d5c8632c81ce5/.tmp/colfamily11/cf6d9387abc845c69d3b2980cf4c7076 as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7e4837322188444dc48d5c8632c81ce5/colfamily11/cf6d9387abc845c69d3b2980cf4c7076 2024-11-08T13:31:57,086 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7e4837322188444dc48d5c8632c81ce5/colfamily11/cf6d9387abc845c69d3b2980cf4c7076, entries=703, sequenceid=5, filesize=38.3 K 2024-11-08T13:31:57,087 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 7e4837322188444dc48d5c8632c81ce5 in 35ms, sequenceid=5, compaction requested=false 2024-11-08T13:31:57,091 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7e4837322188444dc48d5c8632c81ce5/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-08T13:31:57,091 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5. 2024-11-08T13:31:57,091 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7e4837322188444dc48d5c8632c81ce5: Waiting for close lock at 1731072717052Running coprocessor pre-close hooks at 1731072717052Disabling compacts and flushes for region at 1731072717052Disabling writes for close at 1731072717052Obtaining lock to block concurrent updates at 1731072717053 (+1 ms)Preparing flush snapshotting stores in 7e4837322188444dc48d5c8632c81ce5 at 1731072717053Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072717053Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5. at 1731072717053Flushing 7e4837322188444dc48d5c8632c81ce5/colfamily11: creating writer at 1731072717054 (+1 ms)Flushing 7e4837322188444dc48d5c8632c81ce5/colfamily11: appending metadata at 1731072717068 (+14 ms)Flushing 7e4837322188444dc48d5c8632c81ce5/colfamily11: closing flushed file at 1731072717069 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5514d731: reopening flushed file at 1731072717080 (+11 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 7e4837322188444dc48d5c8632c81ce5 in 35ms, sequenceid=5, compaction requested=false at 1731072717087 (+7 ms)Writing region close event to WAL at 1731072717088 (+1 ms)Running coprocessor post-close hooks at 1731072717091 (+3 ms)Closed at 1731072717091 2024-11-08T13:31:57,091 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731072527108.7e4837322188444dc48d5c8632c81ce5. 2024-11-08T13:31:57,091 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5970d249471986ac22ae194d250f9a38, disabling compactions & flushes 2024-11-08T13:31:57,091 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38. 2024-11-08T13:31:57,091 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38. 2024-11-08T13:31:57,091 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38. after waiting 0 ms 2024-11-08T13:31:57,092 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38. 2024-11-08T13:31:57,095 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5970d249471986ac22ae194d250f9a38/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-08T13:31:57,096 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38. 2024-11-08T13:31:57,096 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5970d249471986ac22ae194d250f9a38: Waiting for close lock at 1731072717091Running coprocessor pre-close hooks at 1731072717091Disabling compacts and flushes for region at 1731072717091Disabling writes for close at 1731072717091Writing region close event to WAL at 1731072717092 (+1 ms)Running coprocessor post-close hooks at 1731072717096 (+4 ms)Closed at 1731072717096 2024-11-08T13:31:57,096 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731072527108.5970d249471986ac22ae194d250f9a38. 2024-11-08T13:31:57,096 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7c8fbd70358d0bf3e3f164569432ecfe, disabling compactions & flushes 2024-11-08T13:31:57,096 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe. 2024-11-08T13:31:57,096 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe. 2024-11-08T13:31:57,096 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe. after waiting 0 ms 2024-11-08T13:31:57,096 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe. 2024-11-08T13:31:57,099 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7c8fbd70358d0bf3e3f164569432ecfe/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-08T13:31:57,100 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe. 2024-11-08T13:31:57,101 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7c8fbd70358d0bf3e3f164569432ecfe: Waiting for close lock at 1731072717096Running coprocessor pre-close hooks at 1731072717096Disabling compacts and flushes for region at 1731072717096Disabling writes for close at 1731072717096Writing region close event to WAL at 1731072717097 (+1 ms)Running coprocessor post-close hooks at 1731072717100 (+3 ms)Closed at 1731072717100 2024-11-08T13:31:57,101 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731072527108.7c8fbd70358d0bf3e3f164569432ecfe. 2024-11-08T13:31:57,105 DEBUG [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(1351): Waiting on 0f0799092106200541bc0c416b0bdbb5, 286367177ffe2bf304003f3b829326ac, 4fbf6f08f931761acf639331e9fd9170, 5ab60b8cfddafd2fcce990e602b31aef, 5ef0046eead8230ad7213c161e3532bd, 99abbd965e76aa0c5fe5a1011e45a9ae, c02aa10a67704d3f15deae347223dc13, c4e1216db90b6f396355f2b6bc308ffe, c6699738a1be7c8647568269e341753c, e178bdc5865404fdcd9e707b6918569c, e1c2472f75aa6a0ed9ba5017889bd71b, fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:31:57,105 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(976): stopping server 6ccfea20c990,35367,1731072525051; all regions closed. 2024-11-08T13:31:57,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741865_1041 (size=15752) 2024-11-08T13:31:57,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741865_1041 (size=15752) 2024-11-08T13:31:57,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741865_1041 (size=15752) 2024-11-08T13:31:57,111 DEBUG [RS:2;6ccfea20c990:35367 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/oldWALs 2024-11-08T13:31:57,111 INFO [RS:2;6ccfea20c990:35367 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6ccfea20c990%2C35367%2C1731072525051.meta:.meta(num 1731072712125) 2024-11-08T13:31:57,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741833_1009 (size=314313) 2024-11-08T13:31:57,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741833_1009 (size=314313) 2024-11-08T13:31:57,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741833_1009 (size=314313) 2024-11-08T13:31:57,117 DEBUG [RS:2;6ccfea20c990:35367 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/oldWALs 2024-11-08T13:31:57,117 INFO [RS:2;6ccfea20c990:35367 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6ccfea20c990%2C35367%2C1731072525051:(num 1731072526271) 2024-11-08T13:31:57,117 DEBUG [RS:2;6ccfea20c990:35367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:31:57,117 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T13:31:57,117 INFO [RS:2;6ccfea20c990:35367 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T13:31:57,117 INFO [RS:2;6ccfea20c990:35367 {}] hbase.ChoreService(370): Chore service for: regionserver/6ccfea20c990:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T13:31:57,118 INFO [RS:2;6ccfea20c990:35367 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T13:31:57,118 INFO [regionserver/6ccfea20c990:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T13:31:57,118 INFO [RS:2;6ccfea20c990:35367 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35367 2024-11-08T13:31:57,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6ccfea20c990,35367,1731072525051 2024-11-08T13:31:57,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T13:31:57,119 INFO [RS:2;6ccfea20c990:35367 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T13:31:57,120 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6ccfea20c990,35367,1731072525051] 2024-11-08T13:31:57,121 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6ccfea20c990,35367,1731072525051 already deleted, retry=false 2024-11-08T13:31:57,121 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6ccfea20c990,35367,1731072525051 expired; onlineServers=1 2024-11-08T13:31:57,130 INFO [regionserver/6ccfea20c990:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-08T13:31:57,130 INFO [regionserver/6ccfea20c990:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-08T13:31:57,211 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/286367177ffe2bf304003f3b829326ac/.tmp/colfamily11/115c8686a7924c969e52ef77ca0981ce 2024-11-08T13:31:57,219 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/286367177ffe2bf304003f3b829326ac/.tmp/colfamily11/115c8686a7924c969e52ef77ca0981ce as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/286367177ffe2bf304003f3b829326ac/colfamily11/115c8686a7924c969e52ef77ca0981ce 2024-11-08T13:31:57,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:31:57,221 INFO [RS:2;6ccfea20c990:35367 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T13:31:57,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35367-0x1001f85d9a10003, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:31:57,221 INFO [RS:2;6ccfea20c990:35367 {}] regionserver.HRegionServer(1031): Exiting; stopping=6ccfea20c990,35367,1731072525051; zookeeper connection closed. 2024-11-08T13:31:57,221 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@725869a4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@725869a4 2024-11-08T13:31:57,226 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/286367177ffe2bf304003f3b829326ac/colfamily11/115c8686a7924c969e52ef77ca0981ce, entries=703, sequenceid=5, filesize=38.3 K 2024-11-08T13:31:57,227 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 286367177ffe2bf304003f3b829326ac in 445ms, sequenceid=5, compaction requested=false 2024-11-08T13:31:57,232 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/286367177ffe2bf304003f3b829326ac/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-08T13:31:57,233 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac. 2024-11-08T13:31:57,233 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 286367177ffe2bf304003f3b829326ac: Waiting for close lock at 1731072716782Running coprocessor pre-close hooks at 1731072716782Disabling compacts and flushes for region at 1731072716782Disabling writes for close at 1731072716782Obtaining lock to block concurrent updates at 1731072716782Preparing flush snapshotting stores in 286367177ffe2bf304003f3b829326ac at 1731072716782Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072716783 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac. at 1731072716783Flushing 286367177ffe2bf304003f3b829326ac/colfamily11: creating writer at 1731072716783Flushing 286367177ffe2bf304003f3b829326ac/colfamily11: appending metadata at 1731072716798 (+15 ms)Flushing 286367177ffe2bf304003f3b829326ac/colfamily11: closing flushed file at 1731072716798Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63ee6df0: reopening flushed file at 1731072717218 (+420 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 286367177ffe2bf304003f3b829326ac in 445ms, sequenceid=5, compaction requested=false at 1731072717227 (+9 ms)Writing region close event to WAL at 1731072717228 (+1 ms)Running coprocessor post-close hooks at 1731072717233 (+5 ms)Closed at 1731072717233 2024-11-08T13:31:57,233 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731072527108.286367177ffe2bf304003f3b829326ac. 2024-11-08T13:31:57,233 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c02aa10a67704d3f15deae347223dc13, disabling compactions & flushes 2024-11-08T13:31:57,233 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13. 2024-11-08T13:31:57,234 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13. 2024-11-08T13:31:57,234 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13. after waiting 0 ms 2024-11-08T13:31:57,234 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13. 2024-11-08T13:31:57,234 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c02aa10a67704d3f15deae347223dc13 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:57,253 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c02aa10a67704d3f15deae347223dc13/.tmp/colfamily11/b3264739f1bd4b22ad63f3b5365dc02b is 52, key is qqq/colfamily11:colfamily11/1731072530885/Put/seqid=0 2024-11-08T13:31:57,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741898_1074 (size=39263) 2024-11-08T13:31:57,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741898_1074 (size=39263) 2024-11-08T13:31:57,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741898_1074 (size=39263) 2024-11-08T13:31:57,260 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c02aa10a67704d3f15deae347223dc13/.tmp/colfamily11/b3264739f1bd4b22ad63f3b5365dc02b 2024-11-08T13:31:57,265 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c02aa10a67704d3f15deae347223dc13/.tmp/colfamily11/b3264739f1bd4b22ad63f3b5365dc02b as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c02aa10a67704d3f15deae347223dc13/colfamily11/b3264739f1bd4b22ad63f3b5365dc02b 2024-11-08T13:31:57,271 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c02aa10a67704d3f15deae347223dc13/colfamily11/b3264739f1bd4b22ad63f3b5365dc02b, entries=703, sequenceid=5, filesize=38.3 K 2024-11-08T13:31:57,272 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for c02aa10a67704d3f15deae347223dc13 in 38ms, sequenceid=5, compaction requested=false 2024-11-08T13:31:57,276 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c02aa10a67704d3f15deae347223dc13/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-08T13:31:57,276 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13. 2024-11-08T13:31:57,276 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c02aa10a67704d3f15deae347223dc13: Waiting for close lock at 1731072717233Running coprocessor pre-close hooks at 1731072717233Disabling compacts and flushes for region at 1731072717233Disabling writes for close at 1731072717234 (+1 ms)Obtaining lock to block concurrent updates at 1731072717234Preparing flush snapshotting stores in c02aa10a67704d3f15deae347223dc13 at 1731072717234Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072717234Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13. at 1731072717235 (+1 ms)Flushing c02aa10a67704d3f15deae347223dc13/colfamily11: creating writer at 1731072717235Flushing c02aa10a67704d3f15deae347223dc13/colfamily11: appending metadata at 1731072717253 (+18 ms)Flushing c02aa10a67704d3f15deae347223dc13/colfamily11: closing flushed file at 1731072717253Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@504302f9: reopening flushed file at 1731072717265 (+12 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for c02aa10a67704d3f15deae347223dc13 in 38ms, sequenceid=5, compaction requested=false at 1731072717272 (+7 ms)Writing region close event to WAL at 1731072717273 (+1 ms)Running coprocessor post-close hooks at 1731072717276 (+3 ms)Closed at 1731072717276 2024-11-08T13:31:57,277 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731072527108.c02aa10a67704d3f15deae347223dc13. 2024-11-08T13:31:57,277 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e1c2472f75aa6a0ed9ba5017889bd71b, disabling compactions & flushes 2024-11-08T13:31:57,277 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b. 2024-11-08T13:31:57,277 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b. 2024-11-08T13:31:57,277 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b. after waiting 0 ms 2024-11-08T13:31:57,277 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b. 2024-11-08T13:31:57,277 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e1c2472f75aa6a0ed9ba5017889bd71b 3/3 column families, dataSize=48 B heapSize=888 B 2024-11-08T13:31:57,291 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e1c2472f75aa6a0ed9ba5017889bd71b/.tmp/colfamily11/1b7c045e868245b88bdcc8234ac282aa is 52, key is zzz/colfamily11:colfamily11/1731072530966/Put/seqid=0 2024-11-08T13:31:57,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741899_1075 (size=4857) 2024-11-08T13:31:57,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741899_1075 (size=4857) 2024-11-08T13:31:57,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741899_1075 (size=4857) 2024-11-08T13:31:57,298 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e1c2472f75aa6a0ed9ba5017889bd71b/.tmp/colfamily11/1b7c045e868245b88bdcc8234ac282aa 2024-11-08T13:31:57,305 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e1c2472f75aa6a0ed9ba5017889bd71b/.tmp/colfamily11/1b7c045e868245b88bdcc8234ac282aa as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e1c2472f75aa6a0ed9ba5017889bd71b/colfamily11/1b7c045e868245b88bdcc8234ac282aa 2024-11-08T13:31:57,305 DEBUG [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(1351): Waiting on 0f0799092106200541bc0c416b0bdbb5, 4fbf6f08f931761acf639331e9fd9170, 5ab60b8cfddafd2fcce990e602b31aef, 5ef0046eead8230ad7213c161e3532bd, 99abbd965e76aa0c5fe5a1011e45a9ae, c4e1216db90b6f396355f2b6bc308ffe, c6699738a1be7c8647568269e341753c, e178bdc5865404fdcd9e707b6918569c, e1c2472f75aa6a0ed9ba5017889bd71b, fcf86fec97b560cab0f86ca4727e4e57 2024-11-08T13:31:57,312 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e1c2472f75aa6a0ed9ba5017889bd71b/colfamily11/1b7c045e868245b88bdcc8234ac282aa, entries=1, sequenceid=5, filesize=4.7 K 2024-11-08T13:31:57,313 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~48 B/48, heapSize ~360 B/360, currentSize=0 B/0 for e1c2472f75aa6a0ed9ba5017889bd71b in 36ms, sequenceid=5, compaction requested=false 2024-11-08T13:31:57,316 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e1c2472f75aa6a0ed9ba5017889bd71b/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-08T13:31:57,317 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b. 2024-11-08T13:31:57,317 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e1c2472f75aa6a0ed9ba5017889bd71b: Waiting for close lock at 1731072717277Running coprocessor pre-close hooks at 1731072717277Disabling compacts and flushes for region at 1731072717277Disabling writes for close at 1731072717277Obtaining lock to block concurrent updates at 1731072717277Preparing flush snapshotting stores in e1c2472f75aa6a0ed9ba5017889bd71b at 1731072717277Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b., syncing WAL and waiting on mvcc, flushsize=dataSize=48, getHeapSize=840, getOffHeapSize=0, getCellsCount=1 at 1731072717277Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b. at 1731072717278 (+1 ms)Flushing e1c2472f75aa6a0ed9ba5017889bd71b/colfamily11: creating writer at 1731072717278Flushing e1c2472f75aa6a0ed9ba5017889bd71b/colfamily11: appending metadata at 1731072717291 (+13 ms)Flushing e1c2472f75aa6a0ed9ba5017889bd71b/colfamily11: closing flushed file at 1731072717291Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@753c00bb: reopening flushed file at 1731072717304 (+13 ms)Finished flush of dataSize ~48 B/48, heapSize ~360 B/360, currentSize=0 B/0 for e1c2472f75aa6a0ed9ba5017889bd71b in 36ms, sequenceid=5, compaction requested=false at 1731072717313 (+9 ms)Writing region close event to WAL at 1731072717313Running coprocessor post-close hooks at 1731072717317 (+4 ms)Closed at 1731072717317 2024-11-08T13:31:57,317 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731072527108.e1c2472f75aa6a0ed9ba5017889bd71b. 2024-11-08T13:31:57,317 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e178bdc5865404fdcd9e707b6918569c, disabling compactions & flushes 2024-11-08T13:31:57,317 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c. 2024-11-08T13:31:57,317 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c. 2024-11-08T13:31:57,317 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c. after waiting 0 ms 2024-11-08T13:31:57,317 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c. 2024-11-08T13:31:57,317 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e178bdc5865404fdcd9e707b6918569c 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:57,332 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e178bdc5865404fdcd9e707b6918569c/.tmp/colfamily11/5cdc3d3594e945f798559e34aa80b26f is 52, key is lll/colfamily11:colfamily11/1731072530812/Put/seqid=0 2024-11-08T13:31:57,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741900_1076 (size=39263) 2024-11-08T13:31:57,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741900_1076 (size=39263) 2024-11-08T13:31:57,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741900_1076 (size=39263) 2024-11-08T13:31:57,339 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e178bdc5865404fdcd9e707b6918569c/.tmp/colfamily11/5cdc3d3594e945f798559e34aa80b26f 2024-11-08T13:31:57,345 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e178bdc5865404fdcd9e707b6918569c/.tmp/colfamily11/5cdc3d3594e945f798559e34aa80b26f as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e178bdc5865404fdcd9e707b6918569c/colfamily11/5cdc3d3594e945f798559e34aa80b26f 2024-11-08T13:31:57,351 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e178bdc5865404fdcd9e707b6918569c/colfamily11/5cdc3d3594e945f798559e34aa80b26f, entries=703, sequenceid=5, filesize=38.3 K 2024-11-08T13:31:57,352 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for e178bdc5865404fdcd9e707b6918569c in 35ms, sequenceid=5, compaction requested=false 2024-11-08T13:31:57,356 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e178bdc5865404fdcd9e707b6918569c/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-08T13:31:57,356 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c. 2024-11-08T13:31:57,356 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e178bdc5865404fdcd9e707b6918569c: Waiting for close lock at 1731072717317Running coprocessor pre-close hooks at 1731072717317Disabling compacts and flushes for region at 1731072717317Disabling writes for close at 1731072717317Obtaining lock to block concurrent updates at 1731072717317Preparing flush snapshotting stores in e178bdc5865404fdcd9e707b6918569c at 1731072717317Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072717318 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c. at 1731072717318Flushing e178bdc5865404fdcd9e707b6918569c/colfamily11: creating writer at 1731072717318Flushing e178bdc5865404fdcd9e707b6918569c/colfamily11: appending metadata at 1731072717332 (+14 ms)Flushing e178bdc5865404fdcd9e707b6918569c/colfamily11: closing flushed file at 1731072717332Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e4a4cb5: reopening flushed file at 1731072717344 (+12 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for e178bdc5865404fdcd9e707b6918569c in 35ms, sequenceid=5, compaction requested=false at 1731072717352 (+8 ms)Writing region close event to WAL at 1731072717353 (+1 ms)Running coprocessor post-close hooks at 1731072717356 (+3 ms)Closed at 1731072717356 2024-11-08T13:31:57,356 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731072527108.e178bdc5865404fdcd9e707b6918569c. 2024-11-08T13:31:57,357 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c4e1216db90b6f396355f2b6bc308ffe, disabling compactions & flushes 2024-11-08T13:31:57,357 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe. 2024-11-08T13:31:57,357 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe. 2024-11-08T13:31:57,357 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe. after waiting 0 ms 2024-11-08T13:31:57,357 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe. 2024-11-08T13:31:57,360 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c4e1216db90b6f396355f2b6bc308ffe/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-08T13:31:57,361 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe. 2024-11-08T13:31:57,361 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c4e1216db90b6f396355f2b6bc308ffe: Waiting for close lock at 1731072717357Running coprocessor pre-close hooks at 1731072717357Disabling compacts and flushes for region at 1731072717357Disabling writes for close at 1731072717357Writing region close event to WAL at 1731072717357Running coprocessor post-close hooks at 1731072717361 (+4 ms)Closed at 1731072717361 2024-11-08T13:31:57,361 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731072527108.c4e1216db90b6f396355f2b6bc308ffe. 2024-11-08T13:31:57,361 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4fbf6f08f931761acf639331e9fd9170, disabling compactions & flushes 2024-11-08T13:31:57,361 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170. 2024-11-08T13:31:57,361 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170. 2024-11-08T13:31:57,361 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170. after waiting 0 ms 2024-11-08T13:31:57,361 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170. 2024-11-08T13:31:57,361 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 4fbf6f08f931761acf639331e9fd9170 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:57,376 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4fbf6f08f931761acf639331e9fd9170/.tmp/colfamily11/249092112b7f475599198895d1b0a093 is 52, key is www/colfamily11:colfamily11/1731072530959/Put/seqid=0 2024-11-08T13:31:57,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741901_1077 (size=39263) 2024-11-08T13:31:57,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741901_1077 (size=39263) 2024-11-08T13:31:57,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741901_1077 (size=39263) 2024-11-08T13:31:57,383 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4fbf6f08f931761acf639331e9fd9170/.tmp/colfamily11/249092112b7f475599198895d1b0a093 2024-11-08T13:31:57,389 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4fbf6f08f931761acf639331e9fd9170/.tmp/colfamily11/249092112b7f475599198895d1b0a093 as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4fbf6f08f931761acf639331e9fd9170/colfamily11/249092112b7f475599198895d1b0a093 2024-11-08T13:31:57,395 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4fbf6f08f931761acf639331e9fd9170/colfamily11/249092112b7f475599198895d1b0a093, entries=703, sequenceid=5, filesize=38.3 K 2024-11-08T13:31:57,396 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 4fbf6f08f931761acf639331e9fd9170 in 35ms, sequenceid=5, compaction requested=false 2024-11-08T13:31:57,400 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4fbf6f08f931761acf639331e9fd9170/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-08T13:31:57,400 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170. 2024-11-08T13:31:57,400 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4fbf6f08f931761acf639331e9fd9170: Waiting for close lock at 1731072717361Running coprocessor pre-close hooks at 1731072717361Disabling compacts and flushes for region at 1731072717361Disabling writes for close at 1731072717361Obtaining lock to block concurrent updates at 1731072717361Preparing flush snapshotting stores in 4fbf6f08f931761acf639331e9fd9170 at 1731072717361Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072717362 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170. at 1731072717362Flushing 4fbf6f08f931761acf639331e9fd9170/colfamily11: creating writer at 1731072717362Flushing 4fbf6f08f931761acf639331e9fd9170/colfamily11: appending metadata at 1731072717376 (+14 ms)Flushing 4fbf6f08f931761acf639331e9fd9170/colfamily11: closing flushed file at 1731072717376Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cb72fe3: reopening flushed file at 1731072717388 (+12 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 4fbf6f08f931761acf639331e9fd9170 in 35ms, sequenceid=5, compaction requested=false at 1731072717396 (+8 ms)Writing region close event to WAL at 1731072717397 (+1 ms)Running coprocessor post-close hooks at 1731072717400 (+3 ms)Closed at 1731072717400 2024-11-08T13:31:57,400 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731072527108.4fbf6f08f931761acf639331e9fd9170. 2024-11-08T13:31:57,401 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0f0799092106200541bc0c416b0bdbb5, disabling compactions & flushes 2024-11-08T13:31:57,401 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5. 2024-11-08T13:31:57,401 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5. 2024-11-08T13:31:57,401 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5. after waiting 0 ms 2024-11-08T13:31:57,401 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5. 2024-11-08T13:31:57,401 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0f0799092106200541bc0c416b0bdbb5 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:57,416 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0f0799092106200541bc0c416b0bdbb5/.tmp/colfamily11/28856fb434ab44a1b63c7f73a470b9b0 is 52, key is eee/colfamily11:colfamily11/1731072530708/Put/seqid=0 2024-11-08T13:31:57,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741902_1078 (size=39263) 2024-11-08T13:31:57,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741902_1078 (size=39263) 2024-11-08T13:31:57,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741902_1078 (size=39263) 2024-11-08T13:31:57,423 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0f0799092106200541bc0c416b0bdbb5/.tmp/colfamily11/28856fb434ab44a1b63c7f73a470b9b0 2024-11-08T13:31:57,430 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0f0799092106200541bc0c416b0bdbb5/.tmp/colfamily11/28856fb434ab44a1b63c7f73a470b9b0 as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0f0799092106200541bc0c416b0bdbb5/colfamily11/28856fb434ab44a1b63c7f73a470b9b0 2024-11-08T13:31:57,436 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0f0799092106200541bc0c416b0bdbb5/colfamily11/28856fb434ab44a1b63c7f73a470b9b0, entries=703, sequenceid=5, filesize=38.3 K 2024-11-08T13:31:57,437 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 0f0799092106200541bc0c416b0bdbb5 in 36ms, sequenceid=5, compaction requested=false 2024-11-08T13:31:57,441 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0f0799092106200541bc0c416b0bdbb5/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-08T13:31:57,442 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5. 2024-11-08T13:31:57,442 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0f0799092106200541bc0c416b0bdbb5: Waiting for close lock at 1731072717401Running coprocessor pre-close hooks at 1731072717401Disabling compacts and flushes for region at 1731072717401Disabling writes for close at 1731072717401Obtaining lock to block concurrent updates at 1731072717401Preparing flush snapshotting stores in 0f0799092106200541bc0c416b0bdbb5 at 1731072717401Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072717401Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5. at 1731072717402 (+1 ms)Flushing 0f0799092106200541bc0c416b0bdbb5/colfamily11: creating writer at 1731072717402Flushing 0f0799092106200541bc0c416b0bdbb5/colfamily11: appending metadata at 1731072717415 (+13 ms)Flushing 0f0799092106200541bc0c416b0bdbb5/colfamily11: closing flushed file at 1731072717415Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67573447: reopening flushed file at 1731072717429 (+14 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 0f0799092106200541bc0c416b0bdbb5 in 36ms, sequenceid=5, compaction requested=false at 1731072717437 (+8 ms)Writing region close event to WAL at 1731072717438 (+1 ms)Running coprocessor post-close hooks at 1731072717442 (+4 ms)Closed at 1731072717442 2024-11-08T13:31:57,442 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731072527108.0f0799092106200541bc0c416b0bdbb5. 2024-11-08T13:31:57,442 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c6699738a1be7c8647568269e341753c, disabling compactions & flushes 2024-11-08T13:31:57,442 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c. 2024-11-08T13:31:57,442 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c. 2024-11-08T13:31:57,442 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c. after waiting 0 ms 2024-11-08T13:31:57,442 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c. 2024-11-08T13:31:57,442 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c6699738a1be7c8647568269e341753c 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:57,458 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c6699738a1be7c8647568269e341753c/.tmp/colfamily11/8aa176de409747fb80209349dd9de010 is 52, key is aaa/colfamily11:colfamily11/1731072530655/Put/seqid=0 2024-11-08T13:31:57,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741903_1079 (size=39263) 2024-11-08T13:31:57,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741903_1079 (size=39263) 2024-11-08T13:31:57,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741903_1079 (size=39263) 2024-11-08T13:31:57,465 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c6699738a1be7c8647568269e341753c/.tmp/colfamily11/8aa176de409747fb80209349dd9de010 2024-11-08T13:31:57,472 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c6699738a1be7c8647568269e341753c/.tmp/colfamily11/8aa176de409747fb80209349dd9de010 as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c6699738a1be7c8647568269e341753c/colfamily11/8aa176de409747fb80209349dd9de010 2024-11-08T13:31:57,477 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c6699738a1be7c8647568269e341753c/colfamily11/8aa176de409747fb80209349dd9de010, entries=703, sequenceid=5, filesize=38.3 K 2024-11-08T13:31:57,478 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for c6699738a1be7c8647568269e341753c in 36ms, sequenceid=5, compaction requested=false 2024-11-08T13:31:57,483 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c6699738a1be7c8647568269e341753c/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-08T13:31:57,483 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c. 2024-11-08T13:31:57,483 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c6699738a1be7c8647568269e341753c: Waiting for close lock at 1731072717442Running coprocessor pre-close hooks at 1731072717442Disabling compacts and flushes for region at 1731072717442Disabling writes for close at 1731072717442Obtaining lock to block concurrent updates at 1731072717442Preparing flush snapshotting stores in c6699738a1be7c8647568269e341753c at 1731072717442Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072717443 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c. at 1731072717443Flushing c6699738a1be7c8647568269e341753c/colfamily11: creating writer at 1731072717443Flushing c6699738a1be7c8647568269e341753c/colfamily11: appending metadata at 1731072717458 (+15 ms)Flushing c6699738a1be7c8647568269e341753c/colfamily11: closing flushed file at 1731072717458Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5008027b: reopening flushed file at 1731072717471 (+13 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for c6699738a1be7c8647568269e341753c in 36ms, sequenceid=5, compaction requested=false at 1731072717478 (+7 ms)Writing region close event to WAL at 1731072717479 (+1 ms)Running coprocessor post-close hooks at 1731072717483 (+4 ms)Closed at 1731072717483 2024-11-08T13:31:57,483 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731072527108.c6699738a1be7c8647568269e341753c. 2024-11-08T13:31:57,484 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fcf86fec97b560cab0f86ca4727e4e57, disabling compactions & flushes 2024-11-08T13:31:57,484 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57. 2024-11-08T13:31:57,484 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57. 2024-11-08T13:31:57,484 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57. after waiting 0 ms 2024-11-08T13:31:57,484 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57. 2024-11-08T13:31:57,488 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/fcf86fec97b560cab0f86ca4727e4e57/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-08T13:31:57,488 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57. 2024-11-08T13:31:57,488 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fcf86fec97b560cab0f86ca4727e4e57: Waiting for close lock at 1731072717484Running coprocessor pre-close hooks at 1731072717484Disabling compacts and flushes for region at 1731072717484Disabling writes for close at 1731072717484Writing region close event to WAL at 1731072717484Running coprocessor post-close hooks at 1731072717488 (+4 ms)Closed at 1731072717488 2024-11-08T13:31:57,488 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731072527108.fcf86fec97b560cab0f86ca4727e4e57. 2024-11-08T13:31:57,489 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5ef0046eead8230ad7213c161e3532bd, disabling compactions & flushes 2024-11-08T13:31:57,489 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd. 2024-11-08T13:31:57,489 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd. 2024-11-08T13:31:57,489 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd. after waiting 0 ms 2024-11-08T13:31:57,489 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd. 2024-11-08T13:31:57,493 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ef0046eead8230ad7213c161e3532bd/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-08T13:31:57,493 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd. 2024-11-08T13:31:57,494 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5ef0046eead8230ad7213c161e3532bd: Waiting for close lock at 1731072717489Running coprocessor pre-close hooks at 1731072717489Disabling compacts and flushes for region at 1731072717489Disabling writes for close at 1731072717489Writing region close event to WAL at 1731072717489Running coprocessor post-close hooks at 1731072717493 (+4 ms)Closed at 1731072717493 2024-11-08T13:31:57,494 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731072527108.5ef0046eead8230ad7213c161e3532bd. 2024-11-08T13:31:57,494 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 99abbd965e76aa0c5fe5a1011e45a9ae, disabling compactions & flushes 2024-11-08T13:31:57,494 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae. 2024-11-08T13:31:57,494 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae. 2024-11-08T13:31:57,494 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae. after waiting 0 ms 2024-11-08T13:31:57,494 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae. 2024-11-08T13:31:57,498 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/99abbd965e76aa0c5fe5a1011e45a9ae/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-08T13:31:57,498 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae. 2024-11-08T13:31:57,498 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 99abbd965e76aa0c5fe5a1011e45a9ae: Waiting for close lock at 1731072717494Running coprocessor pre-close hooks at 1731072717494Disabling compacts and flushes for region at 1731072717494Disabling writes for close at 1731072717494Writing region close event to WAL at 1731072717495 (+1 ms)Running coprocessor post-close hooks at 1731072717498 (+3 ms)Closed at 1731072717498 2024-11-08T13:31:57,499 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731072527108.99abbd965e76aa0c5fe5a1011e45a9ae. 2024-11-08T13:31:57,499 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5ab60b8cfddafd2fcce990e602b31aef, disabling compactions & flushes 2024-11-08T13:31:57,499 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef. 2024-11-08T13:31:57,499 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef. 2024-11-08T13:31:57,499 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef. after waiting 0 ms 2024-11-08T13:31:57,499 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef. 2024-11-08T13:31:57,499 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 5ab60b8cfddafd2fcce990e602b31aef 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-08T13:31:57,506 DEBUG [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(1351): Waiting on 5ab60b8cfddafd2fcce990e602b31aef 2024-11-08T13:31:57,520 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ab60b8cfddafd2fcce990e602b31aef/.tmp/colfamily11/052c6433d3df498d9f98ddd3daf16aba is 52, key is ooo/colfamily11:colfamily11/1731072530849/Put/seqid=0 2024-11-08T13:31:57,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741904_1080 (size=39263) 2024-11-08T13:31:57,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741904_1080 (size=39263) 2024-11-08T13:31:57,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741904_1080 (size=39263) 2024-11-08T13:31:57,528 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ab60b8cfddafd2fcce990e602b31aef/.tmp/colfamily11/052c6433d3df498d9f98ddd3daf16aba 2024-11-08T13:31:57,533 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ab60b8cfddafd2fcce990e602b31aef/.tmp/colfamily11/052c6433d3df498d9f98ddd3daf16aba as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ab60b8cfddafd2fcce990e602b31aef/colfamily11/052c6433d3df498d9f98ddd3daf16aba 2024-11-08T13:31:57,539 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ab60b8cfddafd2fcce990e602b31aef/colfamily11/052c6433d3df498d9f98ddd3daf16aba, entries=703, sequenceid=5, filesize=38.3 K 2024-11-08T13:31:57,540 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5ab60b8cfddafd2fcce990e602b31aef in 41ms, sequenceid=5, compaction requested=false 2024-11-08T13:31:57,544 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5ab60b8cfddafd2fcce990e602b31aef/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-08T13:31:57,545 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef. 2024-11-08T13:31:57,545 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5ab60b8cfddafd2fcce990e602b31aef: Waiting for close lock at 1731072717499Running coprocessor pre-close hooks at 1731072717499Disabling compacts and flushes for region at 1731072717499Disabling writes for close at 1731072717499Obtaining lock to block concurrent updates at 1731072717499Preparing flush snapshotting stores in 5ab60b8cfddafd2fcce990e602b31aef at 1731072717499Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731072717499Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef. at 1731072717500 (+1 ms)Flushing 5ab60b8cfddafd2fcce990e602b31aef/colfamily11: creating writer at 1731072717500Flushing 5ab60b8cfddafd2fcce990e602b31aef/colfamily11: appending metadata at 1731072717520 (+20 ms)Flushing 5ab60b8cfddafd2fcce990e602b31aef/colfamily11: closing flushed file at 1731072717520Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49183b28: reopening flushed file at 1731072717533 (+13 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5ab60b8cfddafd2fcce990e602b31aef in 41ms, sequenceid=5, compaction requested=false at 1731072717540 (+7 ms)Writing region close event to WAL at 1731072717541 (+1 ms)Running coprocessor post-close hooks at 1731072717545 (+4 ms)Closed at 1731072717545 2024-11-08T13:31:57,545 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731072527108.5ab60b8cfddafd2fcce990e602b31aef. 2024-11-08T13:31:57,691 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T13:31:57,706 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(976): stopping server 6ccfea20c990,35061,1731072525012; all regions closed. 2024-11-08T13:31:57,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741834_1010 (size=317612) 2024-11-08T13:31:57,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741834_1010 (size=317612) 2024-11-08T13:31:57,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741834_1010 (size=317612) 2024-11-08T13:31:57,711 DEBUG [RS:1;6ccfea20c990:35061 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/oldWALs 2024-11-08T13:31:57,711 INFO [RS:1;6ccfea20c990:35061 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6ccfea20c990%2C35061%2C1731072525012:(num 1731072526271) 2024-11-08T13:31:57,711 DEBUG [RS:1;6ccfea20c990:35061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:31:57,711 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T13:31:57,712 INFO [RS:1;6ccfea20c990:35061 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T13:31:57,712 INFO [RS:1;6ccfea20c990:35061 {}] hbase.ChoreService(370): Chore service for: regionserver/6ccfea20c990:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T13:31:57,712 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T13:31:57,712 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T13:31:57,712 INFO [regionserver/6ccfea20c990:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T13:31:57,712 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T13:31:57,712 INFO [RS:1;6ccfea20c990:35061 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T13:31:57,712 INFO [RS:1;6ccfea20c990:35061 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35061 2024-11-08T13:31:57,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T13:31:57,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6ccfea20c990,35061,1731072525012 2024-11-08T13:31:57,714 INFO [RS:1;6ccfea20c990:35061 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T13:31:57,714 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$364/0x00007f95948c28e0@2a2a2187 rejected from java.util.concurrent.ThreadPoolExecutor@6f28baa6[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 25] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-11-08T13:31:57,714 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6ccfea20c990,35061,1731072525012] 2024-11-08T13:31:57,715 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6ccfea20c990,35061,1731072525012 already deleted, retry=false 2024-11-08T13:31:57,715 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6ccfea20c990,35061,1731072525012 expired; onlineServers=0 2024-11-08T13:31:57,715 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '6ccfea20c990,40009,1731072524365' ***** 2024-11-08T13:31:57,715 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T13:31:57,715 INFO [M:0;6ccfea20c990:40009 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T13:31:57,715 INFO [M:0;6ccfea20c990:40009 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T13:31:57,716 DEBUG [M:0;6ccfea20c990:40009 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T13:31:57,716 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T13:31:57,716 DEBUG [M:0;6ccfea20c990:40009 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T13:31:57,716 DEBUG [master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.small.0-1731072525958 {}] cleaner.HFileCleaner(306): Exit Thread[master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.small.0-1731072525958,5,FailOnTimeoutGroup] 2024-11-08T13:31:57,716 DEBUG [master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.large.0-1731072525955 {}] cleaner.HFileCleaner(306): Exit Thread[master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.large.0-1731072525955,5,FailOnTimeoutGroup] 2024-11-08T13:31:57,716 INFO [M:0;6ccfea20c990:40009 {}] hbase.ChoreService(370): Chore service for: master/6ccfea20c990:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T13:31:57,716 INFO [M:0;6ccfea20c990:40009 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T13:31:57,716 DEBUG [M:0;6ccfea20c990:40009 {}] master.HMaster(1795): Stopping service threads 2024-11-08T13:31:57,716 INFO [M:0;6ccfea20c990:40009 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T13:31:57,716 INFO [M:0;6ccfea20c990:40009 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:31:57,716 INFO [M:0;6ccfea20c990:40009 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T13:31:57,716 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T13:31:57,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T13:31:57,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:31:57,717 DEBUG [M:0;6ccfea20c990:40009 {}] zookeeper.ZKUtil(347): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T13:31:57,717 WARN [M:0;6ccfea20c990:40009 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T13:31:57,718 INFO [M:0;6ccfea20c990:40009 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/.lastflushedseqids 2024-11-08T13:31:57,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741905_1081 (size=2837) 2024-11-08T13:31:57,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741905_1081 (size=2837) 2024-11-08T13:31:57,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741905_1081 (size=2837) 2024-11-08T13:31:57,732 INFO [M:0;6ccfea20c990:40009 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T13:31:57,732 INFO [M:0;6ccfea20c990:40009 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T13:31:57,732 DEBUG [M:0;6ccfea20c990:40009 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T13:31:57,733 INFO [M:0;6ccfea20c990:40009 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:31:57,733 DEBUG [M:0;6ccfea20c990:40009 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:31:57,733 DEBUG [M:0;6ccfea20c990:40009 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T13:31:57,733 DEBUG [M:0;6ccfea20c990:40009 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:31:57,733 INFO [M:0;6ccfea20c990:40009 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=344.65 KB heapSize=405.06 KB 2024-11-08T13:31:57,747 DEBUG [M:0;6ccfea20c990:40009 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ff75d41a11c94710bcf3573bd35235b5 is 82, key is hbase:meta,,1/info:regioninfo/1731072712841/Put/seqid=0 2024-11-08T13:31:57,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741906_1082 (size=6071) 2024-11-08T13:31:57,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741906_1082 (size=6071) 2024-11-08T13:31:57,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741906_1082 (size=6071) 2024-11-08T13:31:57,754 INFO [M:0;6ccfea20c990:40009 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ff75d41a11c94710bcf3573bd35235b5 2024-11-08T13:31:57,776 DEBUG [M:0;6ccfea20c990:40009 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/493e2902e9d548cfb5068f0d4de03e2b is 4105, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731072528417/Put/seqid=0 2024-11-08T13:31:57,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741907_1083 (size=13505) 2024-11-08T13:31:57,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741907_1083 (size=13505) 2024-11-08T13:31:57,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741907_1083 (size=13505) 2024-11-08T13:31:57,783 INFO [M:0;6ccfea20c990:40009 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=343.42 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/493e2902e9d548cfb5068f0d4de03e2b 2024-11-08T13:31:57,807 DEBUG [M:0;6ccfea20c990:40009 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5a3197d024f3467699f0aaf8d714d2f9 is 69, key is 6ccfea20c990,35061,1731072525012/rs:state/1731072526039/Put/seqid=0 2024-11-08T13:31:57,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741908_1084 (size=5442) 2024-11-08T13:31:57,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741908_1084 (size=5442) 2024-11-08T13:31:57,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741908_1084 (size=5442) 2024-11-08T13:31:57,813 INFO [M:0;6ccfea20c990:40009 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5a3197d024f3467699f0aaf8d714d2f9 2024-11-08T13:31:57,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:31:57,815 INFO [RS:1;6ccfea20c990:35061 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T13:31:57,815 INFO [RS:1;6ccfea20c990:35061 {}] regionserver.HRegionServer(1031): Exiting; stopping=6ccfea20c990,35061,1731072525012; zookeeper connection closed. 2024-11-08T13:31:57,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35061-0x1001f85d9a10002, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:31:57,815 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6dff7ea2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6dff7ea2 2024-11-08T13:31:57,816 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-08T13:31:57,818 INFO [M:0;6ccfea20c990:40009 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5a3197d024f3467699f0aaf8d714d2f9 2024-11-08T13:31:57,819 DEBUG [M:0;6ccfea20c990:40009 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ff75d41a11c94710bcf3573bd35235b5 as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ff75d41a11c94710bcf3573bd35235b5 2024-11-08T13:31:57,825 INFO [M:0;6ccfea20c990:40009 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ff75d41a11c94710bcf3573bd35235b5, entries=14, sequenceid=635, filesize=5.9 K 2024-11-08T13:31:57,826 DEBUG [M:0;6ccfea20c990:40009 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/493e2902e9d548cfb5068f0d4de03e2b as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/493e2902e9d548cfb5068f0d4de03e2b 2024-11-08T13:31:57,831 INFO [M:0;6ccfea20c990:40009 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/493e2902e9d548cfb5068f0d4de03e2b, entries=81, sequenceid=635, filesize=13.2 K 2024-11-08T13:31:57,832 DEBUG [M:0;6ccfea20c990:40009 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5a3197d024f3467699f0aaf8d714d2f9 as hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5a3197d024f3467699f0aaf8d714d2f9 2024-11-08T13:31:57,837 INFO [M:0;6ccfea20c990:40009 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5a3197d024f3467699f0aaf8d714d2f9 2024-11-08T13:31:57,837 INFO [M:0;6ccfea20c990:40009 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43827/user/jenkins/test-data/9d135497-3ace-0f97-2c52-380721649c8f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5a3197d024f3467699f0aaf8d714d2f9, entries=3, sequenceid=635, filesize=5.3 K 2024-11-08T13:31:57,838 INFO [M:0;6ccfea20c990:40009 {}] regionserver.HRegion(3140): Finished flush of dataSize ~344.65 KB/352919, heapSize ~404.77 KB/414480, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 105ms, sequenceid=635, compaction requested=false 2024-11-08T13:31:57,839 INFO [M:0;6ccfea20c990:40009 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:31:57,839 DEBUG [M:0;6ccfea20c990:40009 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731072717732Disabling compacts and flushes for region at 1731072717732Disabling writes for close at 1731072717733 (+1 ms)Obtaining lock to block concurrent updates at 1731072717733Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731072717733Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=352919, getHeapSize=414720, getOffHeapSize=0, getCellsCount=800 at 1731072717733Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731072717734 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731072717734Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731072717746 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731072717746Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731072717760 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731072717775 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731072717776 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731072717787 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731072717806 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731072717806Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@173238f9: reopening flushed file at 1731072717818 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dd0a760: reopening flushed file at 1731072717825 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5da4c5a9: reopening flushed file at 1731072717831 (+6 ms)Finished flush of dataSize ~344.65 KB/352919, heapSize ~404.77 KB/414480, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 105ms, sequenceid=635, compaction requested=false at 1731072717838 (+7 ms)Writing region close event to WAL at 1731072717839 (+1 ms)Closed at 1731072717839 2024-11-08T13:31:57,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38513 is added to blk_1073741830_1006 (size=395273) 2024-11-08T13:31:57,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36805 is added to blk_1073741830_1006 (size=395273) 2024-11-08T13:31:57,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36145 is added to blk_1073741830_1006 (size=395273) 2024-11-08T13:31:57,842 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T13:31:57,842 INFO [M:0;6ccfea20c990:40009 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T13:31:57,842 INFO [M:0;6ccfea20c990:40009 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40009 2024-11-08T13:31:57,843 INFO [M:0;6ccfea20c990:40009 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T13:31:57,944 INFO [M:0;6ccfea20c990:40009 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T13:31:57,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:31:57,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40009-0x1001f85d9a10000, quorum=127.0.0.1:50677, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:31:57,948 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@486c2677{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T13:31:57,950 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6738d157{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T13:31:57,950 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T13:31:57,950 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@43bf9dc1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T13:31:57,950 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ed2891b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/hadoop.log.dir/,STOPPED} 2024-11-08T13:31:57,953 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T13:31:57,953 WARN [BP-1581590160-172.17.0.2-1731072521610 heartbeating to localhost/127.0.0.1:43827 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T13:31:57,953 WARN [BP-1581590160-172.17.0.2-1731072521610 heartbeating to localhost/127.0.0.1:43827 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1581590160-172.17.0.2-1731072521610 (Datanode Uuid ccec3428-e338-4216-a1fe-74212cc76f25) service to localhost/127.0.0.1:43827 2024-11-08T13:31:57,953 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T13:31:57,954 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/cluster_fdfcb37c-5278-989a-1d0a-cd8c890d5157/data/data5/current/BP-1581590160-172.17.0.2-1731072521610 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T13:31:57,954 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/cluster_fdfcb37c-5278-989a-1d0a-cd8c890d5157/data/data6/current/BP-1581590160-172.17.0.2-1731072521610 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T13:31:57,955 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T13:31:57,956 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3635c31e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T13:31:57,957 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@78839a64{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T13:31:57,957 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T13:31:57,957 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a292a59{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T13:31:57,957 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@63779849{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/hadoop.log.dir/,STOPPED} 2024-11-08T13:31:57,958 WARN [BP-1581590160-172.17.0.2-1731072521610 heartbeating to localhost/127.0.0.1:43827 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T13:31:57,958 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T13:31:57,958 WARN [BP-1581590160-172.17.0.2-1731072521610 heartbeating to localhost/127.0.0.1:43827 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1581590160-172.17.0.2-1731072521610 (Datanode Uuid ec5066ce-adb8-4b4d-bbb6-cb0173e8c3bc) service to localhost/127.0.0.1:43827 2024-11-08T13:31:57,958 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T13:31:57,959 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/cluster_fdfcb37c-5278-989a-1d0a-cd8c890d5157/data/data3/current/BP-1581590160-172.17.0.2-1731072521610 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T13:31:57,959 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/cluster_fdfcb37c-5278-989a-1d0a-cd8c890d5157/data/data4/current/BP-1581590160-172.17.0.2-1731072521610 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T13:31:57,959 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T13:31:57,965 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4255463d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T13:31:57,966 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5e48d68a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T13:31:57,966 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T13:31:57,966 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@63d82da9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T13:31:57,966 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b71ffd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/hadoop.log.dir/,STOPPED} 2024-11-08T13:31:57,967 WARN [BP-1581590160-172.17.0.2-1731072521610 heartbeating to localhost/127.0.0.1:43827 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T13:31:57,967 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T13:31:57,967 WARN [BP-1581590160-172.17.0.2-1731072521610 heartbeating to localhost/127.0.0.1:43827 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1581590160-172.17.0.2-1731072521610 (Datanode Uuid 66093ed1-7a90-4dc7-ab91-0b1aff75b3f1) service to localhost/127.0.0.1:43827 2024-11-08T13:31:57,967 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T13:31:57,968 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/cluster_fdfcb37c-5278-989a-1d0a-cd8c890d5157/data/data1/current/BP-1581590160-172.17.0.2-1731072521610 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T13:31:57,968 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/cluster_fdfcb37c-5278-989a-1d0a-cd8c890d5157/data/data2/current/BP-1581590160-172.17.0.2-1731072521610 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T13:31:57,968 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T13:31:57,979 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1aa62678{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T13:31:57,979 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1c0c42ad{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T13:31:57,979 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T13:31:57,980 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a372edd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T13:31:57,980 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@58897554{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/16c82b48-3a3c-74f3-65ec-98f66c2568ff/hadoop.log.dir/,STOPPED} 2024-11-08T13:31:57,990 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T13:31:58,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-08T13:31:58,064 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestSCPWithMeta#testRecoveryAndDoubleExecutionOnRsWithMeta Thread=93 (was 11) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/6ccfea20c990:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:43827 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (608921412) connection to localhost/127.0.0.1:43827 from jenkins.hfs.1 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/6ccfea20c990:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/6ccfea20c990:0.logRoller java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180) Potentially hanging thread: IPC Client (608921412) connection to localhost/127.0.0.1:43827 from jenkins.hfs.2 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: region-location-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (608921412) connection to localhost/127.0.0.1:43827 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: master/6ccfea20c990:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: IPC Client (608921412) connection to localhost/127.0.0.1:43827 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: BootstrapNodeManager java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43827 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@19451e47 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43827 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: nioEventLoopGroup-6-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:43827 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-7-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: regionserver/6ccfea20c990:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: nioEventLoopGroup-7-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/6ccfea20c990:0.Chore.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/6ccfea20c990:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43827 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_META-regionserver/6ccfea20c990:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43827 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-7-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:43827 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 285) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=34 (was 100), ProcessCount=11 (was 11), AvailableMemoryMB=2691 (was 3896)